diff --git a/application/HttpUtils.php b/application/HttpUtils.php
index af7cb3712..0e1ce8798 100644
--- a/application/HttpUtils.php
+++ b/application/HttpUtils.php
@@ -27,7 +27,9 @@
function get_http_response($url, $timeout = 30, $maxBytes = 4194304)
{
$urlObj = new Url($url);
- if (! filter_var($url, FILTER_VALIDATE_URL) || ! $urlObj->isHttp()) {
+ $cleanUrl = $urlObj->indToAscii();
+
+ if (! filter_var($cleanUrl, FILTER_VALIDATE_URL) || ! $urlObj->isHttp()) {
return array(array(0 => 'Invalid HTTP Url'), false);
}
@@ -35,22 +37,27 @@ function get_http_response($url, $timeout = 30, $maxBytes = 4194304)
'http' => array(
'method' => 'GET',
'timeout' => $timeout,
- 'user_agent' => 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:23.0)'
- .' Gecko/20100101 Firefox/23.0',
- 'request_fulluri' => true,
+ 'user_agent' => 'Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:45.0)'
+ .' Gecko/20100101 Firefox/45.0',
+ 'accept_language' => substr(setlocale(LC_COLLATE, 0), 0, 2) . ',en-US;q=0.7,en;q=0.3',
)
);
- $context = stream_context_create($options);
stream_context_set_default($options);
+ list($headers, $finalUrl) = get_redirected_headers($cleanUrl);
+ if (! $headers || strpos($headers[0], '200 OK') === false) {
+ $options['http']['request_fulluri'] = true;
+ stream_context_set_default($options);
+ list($headers, $finalUrl) = get_redirected_headers($cleanUrl);
+ }
- list($headers, $finalUrl) = get_redirected_headers($urlObj->cleanup());
if (! $headers || strpos($headers[0], '200 OK') === false) {
return array($headers, false);
}
try {
// TODO: catch Exception in calling code (thumbnailer)
+ $context = stream_context_create($options);
$content = file_get_contents($finalUrl, false, $context, -1, $maxBytes);
} catch (Exception $exc) {
return array(array(0 => 'HTTP Error'), $exc->getMessage());
@@ -60,16 +67,19 @@ function get_http_response($url, $timeout = 30, $maxBytes = 4194304)
}
/**
- * Retrieve HTTP headers, following n redirections (temporary and permanent).
+ * Retrieve HTTP headers, following n redirections (temporary and permanent ones).
*
- * @param string $url initial URL to reach.
- * @param int $redirectionLimit max redirection follow..
+ * @param string $url initial URL to reach.
+ * @param int $redirectionLimit max redirection follow..
*
- * @return array
+ * @return array HTTP headers, or false if it failed.
*/
function get_redirected_headers($url, $redirectionLimit = 3)
{
$headers = get_headers($url, 1);
+ if (!empty($headers['location']) && empty($headers['Location'])) {
+ $headers['Location'] = $headers['location'];
+ }
// Headers found, redirection found, and limit not reached.
if ($redirectionLimit-- > 0
@@ -79,6 +89,7 @@ function get_redirected_headers($url, $redirectionLimit = 3)
$redirection = is_array($headers['Location']) ? end($headers['Location']) : $headers['Location'];
if ($redirection != $url) {
+ $redirection = getAbsoluteUrl($url, $redirection);
return get_redirected_headers($redirection, $redirectionLimit);
}
}
@@ -86,6 +97,35 @@ function get_redirected_headers($url, $redirectionLimit = 3)
return array($headers, $url);
}
+/**
+ * Get an absolute URL from a complete one, and another absolute/relative URL.
+ *
+ * @param string $originalUrl The original complete URL.
+ * @param string $newUrl The new one, absolute or relative.
+ *
+ * @return string Final URL:
+ * - $newUrl if it was already an absolute URL.
+ * - if it was relative, absolute URL from $originalUrl path.
+ */
+function getAbsoluteUrl($originalUrl, $newUrl)
+{
+ $newScheme = parse_url($newUrl, PHP_URL_SCHEME);
+ // Already an absolute URL.
+ if (!empty($newScheme)) {
+ return $newUrl;
+ }
+
+ $parts = parse_url($originalUrl);
+ $final = $parts['scheme'] .'://'. $parts['host'];
+ $final .= (!empty($parts['port'])) ? $parts['port'] : '';
+ $final .= '/';
+ if ($newUrl[0] != '/') {
+ $final .= substr(ltrim($parts['path'], '/'), 0, strrpos($parts['path'], '/'));
+ }
+ $final .= ltrim($newUrl, '/');
+ return $final;
+}
+
/**
* Returns the server's base URL: scheme://domain.tld[:port]
*
diff --git a/application/LinkUtils.php b/application/LinkUtils.php
index d8dc8b5e0..2df76ba8a 100644
--- a/application/LinkUtils.php
+++ b/application/LinkUtils.php
@@ -9,8 +9,8 @@
*/
function html_extract_title($html)
{
- if (preg_match('!
(.*?)!is', $html, $matches)) {
- return trim(str_replace("\n", ' ', $matches[1]));
+ if (preg_match('!(.*?)!is', $html, $matches)) {
+ return trim(str_replace("\n", '', $matches[1]));
}
return false;
}
@@ -70,7 +70,7 @@ function headers_extract_charset($headers)
function html_extract_charset($html)
{
// Get encoding specified in HTML header.
- preg_match('#/]+)"? */?>#Usi', $html, $enc);
+ preg_match('#/]+)["\']? */?>#Usi', $html, $enc);
if (!empty($enc[1])) {
return strtolower($enc[1]);
}
diff --git a/application/Url.php b/application/Url.php
index af38c4d91..260a4f7ef 100644
--- a/application/Url.php
+++ b/application/Url.php
@@ -62,7 +62,20 @@ function add_trailing_slash($url)
{
return $url . (!endsWith($url, '/') ? '/' : '');
}
+/**
+ * Converts an URL with an IDN host to a ASCII one.
+ *
+ * @param string $url Input URL.
+ *
+ * @return string converted URL.
+ */
+function url_with_idn_to_ascii($url)
+{
+ $parts = parse_url($url);
+ $parts['host'] = idn_to_ascii($parts['host']);
+ return (new \http\Url($parts))->toString();
+}
/**
* URL representation and cleanup utilities
*
@@ -220,6 +233,22 @@ public function cleanup()
return $this->toString();
}
+ /**
+ * Converts an URL with an International Domain Name host to a ASCII one.
+ * This requires PHP-intl. If it's not available, just returns this->cleanup().
+ *
+ * @return string converted cleaned up URL.
+ */
+ public function indToAscii()
+ {
+ $out = $this->cleanup();
+ if (! function_exists('idn_to_ascii') || ! isset($this->parts['host'])) {
+ return $out;
+ }
+ $asciiHost = idn_to_ascii($this->parts['host']);
+ return str_replace($this->parts['host'], $asciiHost, $out);
+ }
+
/**
* Get URL scheme.
*
@@ -232,6 +261,18 @@ public function getScheme() {
return $this->parts['scheme'];
}
+ /**
+ * Get URL host.
+ *
+ * @return string the URL host or false if none is provided.
+ */
+ public function getHost() {
+ if (empty($this->parts['host'])) {
+ return false;
+ }
+ return $this->parts['host'];
+ }
+
/**
* Test if the Url is an HTTP one.
*
diff --git a/index.php b/index.php
index dfc00fbd6..41a42cf67 100644
--- a/index.php
+++ b/index.php
@@ -1516,7 +1516,7 @@ function renderPage()
// -------- User want to post a new link: Display link edit form.
if (isset($_GET['post'])) {
- $url = cleanup_url(escape($_GET['post']));
+ $url = cleanup_url($_GET['post']);
$link_is_new = false;
// Check if URL is not already in database (in this case, we will edit the existing link)
@@ -1541,8 +1541,8 @@ function renderPage()
// Extract title.
$title = html_extract_title($content);
// Re-encode title in utf-8 if necessary.
- if (! empty($title) && $charset != 'utf-8') {
- $title = mb_convert_encoding($title, $charset, 'utf-8');
+ if (! empty($title) && strtolower($charset) != 'utf-8') {
+ $title = mb_convert_encoding($title, 'utf-8', $charset);
}
}
}
@@ -1551,6 +1551,8 @@ function renderPage()
$url = '?' . smallHash($linkdate);
$title = 'Note: ';
}
+ $url = escape($url);
+ $title = escape($title);
$link = array(
'linkdate' => $linkdate,
diff --git a/tests/HttpUtils/GetHttpUrlTest.php b/tests/HttpUtils/GetHttpUrlTest.php
index fd2935053..d07c5dbb9 100644
--- a/tests/HttpUtils/GetHttpUrlTest.php
+++ b/tests/HttpUtils/GetHttpUrlTest.php
@@ -35,4 +35,36 @@ public function testGetInvalidRemoteUrl()
$this->assertFalse($headers);
$this->assertFalse($content);
}
+
+ /**
+ * Test getAbsoluteUrl with relative target URL.
+ */
+ public function testGetAbsoluteUrlWithRelative()
+ {
+ $origin = 'http://non.existent/blabla/?test';
+ $target = '/stuff.php';
+
+ $expected = 'http://non.existent/stuff.php';
+ $this->assertEquals($expected, getAbsoluteUrl($origin, $target));
+
+ $target = 'stuff.php';
+ $expected = 'http://non.existent/blabla/stuff.php';
+ $this->assertEquals($expected, getAbsoluteUrl($origin, $target));
+ }
+
+ /**
+ * Test getAbsoluteUrl with absolute target URL.
+ */
+ public function testGetAbsoluteUrlWithAbsolute()
+ {
+ $origin = 'http://non.existent/blabla/?test';
+ $target = 'http://other.url/stuff.php';
+
+ $this->assertEquals($target, getAbsoluteUrl($origin, $target));
+ }
+
+ public function tmpTest()
+ {
+
+ }
}
diff --git a/tests/Url/UrlTest.php b/tests/Url/UrlTest.php
index a64a73eae..5fdc86177 100644
--- a/tests/Url/UrlTest.php
+++ b/tests/Url/UrlTest.php
@@ -181,4 +181,19 @@ function testUrlIsNotHttp()
$url = new Url('ftp://save.tld/mysave');
$this->assertFalse($url->isHttp());
}
+
+ /**
+ * Test IndToAscii.
+ */
+ function testIndToAscii()
+ {
+ $ind = 'http://www.académie-française.fr/';
+ $expected = 'http://www.xn--acadmie-franaise-npb1a.fr/';
+ $url = new Url($ind);
+ $this->assertEquals($expected, $url->indToAscii());
+
+ $notInd = 'http://www.academie-francaise.fr/';
+ $url = new Url($notInd);
+ $this->assertEquals($notInd, $url->indToAscii());
+ }
}