From 3aed0bfdfa3bc8a77e1f2f6bae959c93a40c8d7a Mon Sep 17 00:00:00 2001 From: Nikolay Tumbalev Date: Thu, 13 Feb 2020 12:35:15 +0200 Subject: [PATCH 1/7] Remove selenium server from composer. --- composer.json | 3 +- composer.lock | 86 ++++++++++++++------------------------------------- 2 files changed, 25 insertions(+), 64 deletions(-) diff --git a/composer.json b/composer.json index 53a5723..3d85bf7 100644 --- a/composer.json +++ b/composer.json @@ -1,6 +1,6 @@ { "name": "openbuildings/spiderling", - "description": "Crawl the web with kohana, phantomjs or selenium", + "description": "Crawl the web with kohana or phantomjs.", "license": "BSD-3-Clause", "authors": [ { @@ -15,7 +15,6 @@ }, "require-dev": { "openbuildings/kohana-test-bootstrap": "^0.2", - "se/selenium-server-standalone": "2.40", "phpunit/phpunit": "^4.0|^5.0" }, "autoload": { diff --git a/composer.lock b/composer.lock index c94ac50..823a7f7 100644 --- a/composer.lock +++ b/composer.lock @@ -1,11 +1,10 @@ { "_readme": [ "This file locks the dependencies of your project to a known state", - "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", + "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "hash": "ca10cd033edf47edf34f642431e95554", - "content-hash": "0c34d029dd0d789c01dc7a14a8162b46", + "content-hash": "a70cc1858d0972d18a83618f450bd112", "packages": [ { "name": "openbuildings/environment-backup", @@ -43,7 +42,7 @@ } ], "description": "Backup/restore environment variables: globals, static vars, configs", - "time": "2013-07-24 07:26:52" + "time": "2013-07-24T07:26:52+00:00" }, { "name": "symfony/css-selector", @@ -96,7 +95,7 @@ ], "description": "Symfony CssSelector Component", "homepage": "https://symfony.com", - "time": "2016-06-29 05:31:50" + "time": "2016-06-29T05:31:50+00:00" } ], "packages-dev": [ @@ -152,7 +151,7 @@ "constructor", "instantiate" ], - "time": "2015-06-14 21:17:01" + "time": "2015-06-14T21:17:01+00:00" }, { "name": "kohana/core", @@ -210,7 +209,7 @@ "framework", "kohana" ], - "time": "2016-07-25 13:15:13" + "time": "2016-07-25T13:15:13+00:00" }, { "name": "openbuildings/kohana-test-bootstrap", @@ -252,7 +251,7 @@ } ], "description": "Helper package for testing Kohana modules", - "time": "2016-07-28 14:07:23" + "time": "2016-07-28T14:07:23+00:00" }, { "name": "phpdocumentor/reflection-docblock", @@ -301,7 +300,7 @@ "email": "mike.vanriel@naenius.com" } ], - "time": "2015-02-03 12:10:50" + "time": "2015-02-03T12:10:50+00:00" }, { "name": "phpspec/prophecy", @@ -363,7 +362,7 @@ "spy", "stub" ], - "time": "2016-06-07 08:13:47" + "time": "2016-06-07T08:13:47+00:00" }, { "name": "phpunit/php-code-coverage", @@ -425,7 +424,7 @@ "testing", "xunit" ], - "time": "2015-10-06 15:47:00" + "time": "2015-10-06T15:47:00+00:00" }, { "name": "phpunit/php-file-iterator", @@ -472,7 +471,7 @@ "filesystem", "iterator" ], - "time": "2015-06-21 13:08:43" + "time": "2015-06-21T13:08:43+00:00" }, { "name": "phpunit/php-text-template", @@ -513,7 +512,7 @@ "keywords": [ "template" ], - "time": "2015-06-21 13:50:34" + "time": "2015-06-21T13:50:34+00:00" }, { "name": "phpunit/php-timer", @@ -557,7 +556,7 @@ "keywords": [ "timer" ], - "time": "2016-05-12 18:03:57" + "time": "2016-05-12T18:03:57+00:00" }, { "name": "phpunit/php-token-stream", @@ -606,7 +605,7 @@ "keywords": [ "tokenizer" ], - "time": "2015-09-15 10:49:45" + "time": "2015-09-15T10:49:45+00:00" }, { "name": "phpunit/phpunit", @@ -678,7 +677,7 @@ "testing", "xunit" ], - "time": "2016-07-21 06:48:14" + "time": "2016-07-21T06:48:14+00:00" }, { "name": "phpunit/phpunit-mock-objects", @@ -734,44 +733,7 @@ "mock", "xunit" ], - "time": "2015-10-02 06:51:40" - }, - { - "name": "se/selenium-server-standalone", - "version": "v2.40.0", - "source": { - "type": "git", - "url": "https://github.com/sveneisenschmidt/selenium-server-standalone.git", - "reference": "c3c3b175265c747e3eff7b2beb323f86bd3ee2bb" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/sveneisenschmidt/selenium-server-standalone/zipball/c3c3b175265c747e3eff7b2beb323f86bd3ee2bb", - "reference": "c3c3b175265c747e3eff7b2beb323f86bd3ee2bb", - "shasum": "" - }, - "bin": [ - "composer/bin/selenium-server-standalone" - ], - "type": "library", - "notification-url": "https://packagist.org/downloads/", - "license": [ - "Apache 2.0" - ], - "authors": [ - { - "name": "Sven Eisenschmidt", - "email": "sven.eisenschmidt@gmail.com", - "homepage": "http://unsicherheitsagent.de/" - } - ], - "description": "Composer distribution of Selenium Server Standalone, the browser automation framework. Adds a executable to your composer bin directory.", - "homepage": "https://github.com/sveneisenschmidt/selenium-server-standalone", - "keywords": [ - "selenium", - "testing" - ], - "time": "2014-03-13 13:17:49" + "time": "2015-10-02T06:51:40+00:00" }, { "name": "sebastian/comparator", @@ -835,7 +797,7 @@ "compare", "equality" ], - "time": "2015-07-26 15:48:44" + "time": "2015-07-26T15:48:44+00:00" }, { "name": "sebastian/diff", @@ -887,7 +849,7 @@ "keywords": [ "diff" ], - "time": "2015-12-08 07:14:41" + "time": "2015-12-08T07:14:41+00:00" }, { "name": "sebastian/environment", @@ -937,7 +899,7 @@ "environment", "hhvm" ], - "time": "2016-08-18 05:49:44" + "time": "2016-08-18T05:49:44+00:00" }, { "name": "sebastian/exporter", @@ -1004,7 +966,7 @@ "export", "exporter" ], - "time": "2016-06-17 09:04:28" + "time": "2016-06-17T09:04:28+00:00" }, { "name": "sebastian/global-state", @@ -1055,7 +1017,7 @@ "keywords": [ "global state" ], - "time": "2015-10-12 03:26:01" + "time": "2015-10-12T03:26:01+00:00" }, { "name": "sebastian/recursion-context", @@ -1108,7 +1070,7 @@ ], "description": "Provides functionality to recursively process PHP variables", "homepage": "http://www.github.com/sebastianbergmann/recursion-context", - "time": "2015-11-11 19:50:13" + "time": "2015-11-11T19:50:13+00:00" }, { "name": "sebastian/version", @@ -1143,7 +1105,7 @@ ], "description": "Library that helps with managing the version number of Git-hosted PHP projects", "homepage": "https://github.com/sebastianbergmann/version", - "time": "2015-06-21 13:59:46" + "time": "2015-06-21T13:59:46+00:00" }, { "name": "symfony/yaml", @@ -1192,7 +1154,7 @@ ], "description": "Symfony Yaml Component", "homepage": "https://symfony.com", - "time": "2016-07-17 09:06:15" + "time": "2016-07-17T09:06:15+00:00" } ], "aliases": [], From d668d2eff5f95f0c945e7cdd2cc8d9f76ded3f00 Mon Sep 17 00:00:00 2001 From: Nikolay Tumbalev Date: Thu, 13 Feb 2020 12:35:32 +0200 Subject: [PATCH 2/7] Remove selenium server run from travis. --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7a7f847..7dfbc3e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -57,7 +57,6 @@ before_script: - mkdir -p build/logs - "export DISPLAY=:99.0" - "sh -e /etc/init.d/xvfb start" - - "nohup java -jar vendor/se/selenium-server-standalone/composer/bin/selenium-server-standalone.jar > /dev/null 2> /dev/null &" - sleep 3 script: From cc6b32889633c63350c70414a71ed75dc855fe7c Mon Sep 17 00:00:00 2001 From: Nikolay Tumbalev Date: Thu, 13 Feb 2020 12:35:47 +0200 Subject: [PATCH 3/7] Remove the selenium related files. --- .../Spiderling/Driver/Selenium.php | 434 ------------------ .../Spiderling/Driver/Selenium/Connection.php | 191 -------- .../Spiderling/Exception/Selenium.php | 78 ---- .../tests/Driver/Selenium/ConnectionTest.php | 41 -- tests/tests/Driver/SeleniumTest.php | 288 ------------ 5 files changed, 1032 deletions(-) delete mode 100644 src/Openbuildings/Spiderling/Driver/Selenium.php delete mode 100644 src/Openbuildings/Spiderling/Driver/Selenium/Connection.php delete mode 100644 src/Openbuildings/Spiderling/Exception/Selenium.php delete mode 100644 tests/tests/Driver/Selenium/ConnectionTest.php delete mode 100644 tests/tests/Driver/SeleniumTest.php diff --git a/src/Openbuildings/Spiderling/Driver/Selenium.php b/src/Openbuildings/Spiderling/Driver/Selenium.php deleted file mode 100644 index da52bde..0000000 --- a/src/Openbuildings/Spiderling/Driver/Selenium.php +++ /dev/null @@ -1,434 +0,0 @@ -_base_url = $base_url; - return $this; - } - return $this->_base_url; - } - - /** - * Clear the connection by deleting all cookies - */ - public function clear() - { - $this->connection()->delete('cookie'); - } - - /** - * Getter of the raw content html, this driver does not allow "setting" - * - * @param string $content - * @return string - */ - public function content($content = NULL) - { - return $this->connection()->get('source'); - } - - /** - * Getter / Setter of the Driver_Selenium_Connection object. - * Use this to customize the connection, otherwise a default one on a random port will be used - * - * @param Driver_Selenium_Connection $connection - * @return Driver_Selenium_Connection|Driver_Selenium - */ - public function connection(Driver_Selenium_Connection $connection = NULL) - { - if ($connection !== NULL) - { - $this->_connection = $connection; - return $this; - } - - if ( ! $this->_connection) - { - $this->_connection = new Driver_Selenium_Connection(); - $this->_connection->start(array('browserName' => 'firefox', 'acceptSslCerts' => FALSE)); - } - - return $this->_connection; - } - - /** - * NODE GETTERS - * ===================================== - */ - - /** - * Get the tag name of a Node with id. e.g. DIV, SPAN ... - * @param string $id - * @return string - */ - public function tag_name($id) - { - return $this->connection()->get("element/$id/name"); - } - - /** - * Get the attribute of a Node with id. If the attribute does not exist, returns NULL - * @param string $id - * @param string $name - * @return string - */ - public function attribute($id, $name) - { - return $this->connection()->get("element/$id/attribute/$name"); - } - - /** - * Return the raw html of a Node with id, along with all of its children. - * @param string $id - * @return string - */ - public function html($id) - { - if ($id === NULL) - return $this->content(); - - return $this->execute($id, 'return arguments[0].outerHTML'); - } - - /** - * Return the text of a Node with id, with all the spaces collapsed, similar to browser rendering. - * @param string $id - * @return string - */ - public function text($id) - { - $text = $this->connection()->get("element/$id/text"); - $text = preg_replace('/[ \s\f\n\r\t\v ]+/u', ' ', $text); - return trim($text); - } - - /** - * Return the value of a Node of a form element, e.g. INPUT, TEXTAREA or SELECT - * @param string $id - * @return string - */ - public function value($id) - { - if ($this->tag_name($id) == 'select' AND $this->attribute($id, 'multiple')) - { - $self = $this; - $values = array(); - foreach ($this->all('//option', $id) as $option_id) - { - if ($this->is_selected($option_id)) - { - $values []= $this->value($option_id); - } - } - return $values; - } - else - { - return $this->connection()->get("element/$id/value"); - } - } - - /** - * Check if a Node with id is visible. - * @param string $id - * @return boolean - */ - public function is_visible($id) - { - return $this->connection()->get("element/$id/displayed"); - } - - /** - * Check if a Node with id of an option element is selected - * @param string $id - * @return boolean - */ - public function is_selected($id) - { - return $this->connection()->get("element/$id/selected"); - } - - /** - * Check if a Node with id of an input element (radio or checkbox) is checked - * @param string $id - * @return boolean - */ - public function is_checked($id) - { - return $this->connection()->get("element/$id/selected"); - } - - /** - * Set the value of a Node with id of a form element - * @param string $id - * @param string $value - */ - public function set($id, $value) - { - $tag_name = $this->tag_name($id); - - if ($tag_name == 'textarea') - { - $this->connection()->post("element/$id/clear", array()); - $this->connection()->post("element/$id/value", array('value' => str_split($value))); - } - elseif ($tag_name == 'input') - { - $type = $this->attribute($id, 'type'); - if ($type == 'checkbox' OR $type == 'radio') - { - $this->connection()->post("element/$id/click", array()); - } - else - { - if ($type !== 'file') - { - $this->connection()->post("element/$id/clear", array()); - } - $this->connection()->post("element/$id/value", array('value' => str_split($value))); - } - } - elseif ($tag_name == 'option') - { - $this->connection()->post("element/$id/click", array()); - } - } - - /** - * Set the option value that is selected of a Node of a select element - * @param string $id - * @param string $value - */ - public function select_option($id, $value) - { - $this->connection()->post("element/$id/click", array()); - } - - /** - * Confirm or cancel for the next confirmation dialog - * @param bool $confirm - */ - public function confirm($confirm) - { - if ($confirm) - { - $this->connection()->post('accept_alert', array()); - } - else - { - $this->connection()->post('dismiss_alert', array()); - } - } - - /** - * Get the text of the currently displayed alert / confirm /prompt dialog - * @param bool $confirm - */ - public function alert_text() - { - return $this->connection()->get("alert_text"); - } - - /** - * Click on a Node with id, triggering a link or form submit - * @param string $id - */ - public function click($id) - { - $this->connection()->post("element/$id/click", array()); - } - - /** - * Go to a given url address, use next_query along with the provided query array - * @param string $uri - * @param array $query - */ - public function visit($uri, array $query = array()) - { - $query = array_merge((array) $this->_next_query, (array) $query); - - $this->_next_query = NULL; - - // Check for implicit query string - if (strpos($uri, '?') !== FALSE) - { - $query = array_merge(self::extract_query_from_uri($uri), $query); - $uri = substr($uri, 0, strpos($uri, '?')); - } - - $url = $this->base_url().$uri.($query ? '?'.http_build_query($query) : ''); - - $this->connection()->post('url', array('url' => $url)); - } - - /** - * Get the current path (without host and protocol) - * @return string - */ - public function current_path() - { - $url = parse_url($this->connection()->get('url')); - - return $url['path'].(isset($url['query']) ? '?'.$url['query'] : ''); - } - - /** - * Get the current url - * @return string - */ - public function current_url() - { - return urldecode($this->connection()->get('url')); - } - - /** - * Find all ids of a given XPath - * @param string $xpath - * @param string $parent id of the parent node - * @return array - */ - public function all($xpath, $parent = NULL) - { - $elements = $this->connection()->post(($parent === NULL ? '' : 'element/'.$parent.'/').'elements', array('using' => 'xpath', 'value' => '.'.$xpath)); - - return array_map(function($item){ return $item['ELEMENT'];}, $elements); - } - - /** - * Setter for the next_query variable, to be added to the next visit's query - * @param array $query - */ - public function next_query(array $query) - { - $this->_next_query = $query; - return $this; - } - - /** - * Execute raw javascript. it will be executed in the context of a given node ('this' will point to the node) and the return of the script will be the return of this method - * @param string $id - * @param string $script - * @return mixed - */ - public function execute($id, $script) - { - return $this->connection()->post('execute', array( - 'script' => $script, - 'args' => $id ? array(array('ELEMENT' => $id)) : array() - )); - } - - /** - * Check if a connection is active - * @return boolean - */ - public function is_page_active() - { - return (bool) $this->_connection; - } - - /** - * Move the mouse to a given element, or coordinates, or coordinates relative to a given element - * @param string $id - * @param integer $x - * @param integer $y - */ - public function move_to($id = NULL, $x = NULL, $y = NULL) - { - $this->connection()->post('moveto', array_filter(array( - 'element' => $id, - 'xoffset' => $x, - 'yoffset' => $y - ), function($param) - { - return ($param OR $param === 0); - })); - } - - /** - * Do a screenshot of the current page into a file - * @param string $file - */ - public function screenshot($file) - { - $data = $this->connection()->get('screenshot'); - - file_put_contents($file, base64_decode($data)); - } - - /** - * Return all the current cookies - * @return array - */ - public function cookies() - { - return $this->connection()->get('cookie'); - } - - /** - * Set a cookie. Use parameters to set "expiry", "path", "domain" and "secure" - * @param string $name - * @param mixed $value - * @param array $parameters - */ - public function cookie($name, $value, array $parameters = array()) - { - $parameters = array_merge(array( - 'name' => $name, - 'value' => $value, - 'expiry' => time() + 86400, - ), $parameters); - - return $this->connection()->post('cookie', array('cookie' => $parameters)); - } -} diff --git a/src/Openbuildings/Spiderling/Driver/Selenium/Connection.php b/src/Openbuildings/Spiderling/Driver/Selenium/Connection.php deleted file mode 100644 index 31166ca..0000000 --- a/src/Openbuildings/Spiderling/Driver/Selenium/Connection.php +++ /dev/null @@ -1,191 +0,0 @@ -_server = $server; - return $this; - } - return $this->_server; - } - - public function __construct($server = NULL) - { - if ($server) - { - $this->server($server); - } - } - - /** - * Start a new selenium session, based on passed desired capabilities. - * Reuses the session if possible - * - * @param array $desiredCapabilities - */ - public function start(array $desiredCapabilities = NULL) - { - if ( ! ($this->_session_id = $this->reuse_session())) - { - $this->_session_id = $this->new_session($desiredCapabilities); - } - - $this->_server .= "session/{$this->_session_id}/"; - } - - /** - * Check if a session has been started - * @return boolean - */ - public function is_started() - { - return (bool) $this->_session_id; - } - - /** - * Try to reuse an existing selenium session, return the reused id - * @return string - */ - public function reuse_session() - { - $sessions = $this->get('sessions'); - foreach ($sessions as $session) - { - $id = $session['id']; - try - { - $this->get("session/$id/window_handle"); - return $id; - } - // @codeCoverageIgnoreStart - // This cannot be tested because of selenium bug (can't close main window) - catch (Exception_Selenium $exception) - { - $this->delete("session/$id"); - } - // @codeCoverageIgnoreEnd - } - } - - /** - * Initiate a new session, based on the desired capabilities - * @param array $desiredCapabilities - * @return new session - */ - public function new_session(array $desiredCapabilities = NULL) - { - if ( ! $desiredCapabilities) - { - $desiredCapabilities = array('browserName' => 'firefox'); - } - - $session = $this->post('session', array('desiredCapabilities' => $desiredCapabilities)); - return $session['webdriver.remote.sessionid']; - } - - /** - * Perform a get request to the selenium server - * @param string $command - * @return mixed - */ - public function get($command) - { - return $this->call($command); - } - - /** - * Perform a post request to the selenium server - * @param string $command - * @param array $params - * @return mixed - */ - public function post($command, array $params) - { - $options = array(); - $options[CURLOPT_POST] = TRUE; - - if ($params) { - $options[CURLOPT_POSTFIELDS] = json_encode($params); - } - - return $this->call($command, $options); - } - - /** - * Perform a delete request to the selenium server - * @param string $command - * @return mixed - */ - public function delete($command) - { - $options = array(); - $options[CURLOPT_CUSTOMREQUEST] = 'DELETE'; - - return $this->call($command, $options); - } - - /** - * Perform a request to the selenium server, using curl - * @param string $command - * @param array $options curl options - * @return mixed - */ - public function call($command, array $options = array()) - { - $curl = curl_init(); - $options[CURLOPT_URL] = $this->server().$command; - $options[CURLOPT_RETURNTRANSFER] = TRUE; - $options[CURLOPT_FOLLOWLOCATION] = TRUE; - $options[CURLOPT_HTTPHEADER] = array( - 'Content-Type: application/json;charset=UTF-8', - 'Accept: application/json', - ); - - curl_setopt_array($curl, $options); - - $raw = trim(curl_exec($curl)); - - $result = json_decode($raw, TRUE); - - $error = curl_error($curl); - - curl_close($curl); - - if ($error) - throw new Exception_Driver('Curl ":command" throws exception :error', array(':command' => $command, ':error' => $error)); - - if ($result['status'] == 10) - throw new Exception_Staleelement(); - - if ($result['status'] != 0) - throw new Exception_Selenium($result['status']); - - return $result['value']; - } -} diff --git a/src/Openbuildings/Spiderling/Exception/Selenium.php b/src/Openbuildings/Spiderling/Exception/Selenium.php deleted file mode 100644 index 02caba7..0000000 --- a/src/Openbuildings/Spiderling/Exception/Selenium.php +++ /dev/null @@ -1,78 +0,0 @@ - 'NoSuchDriver', - 7 => 'NoSuchElement', - 8 => 'NoSuchFrame', - 9 => 'UnknownCommand', - 10 => 'StaleElementReference', - 11 => 'ElementNotVisible', - 12 => 'InvalidElementState', - 13 => 'UnknownError', - 15 => 'ElementIsNotSelectable', - 17 => 'JavaScriptError', - 19 => 'XPathLookupError', - 21 => 'Timeout', - 23 => 'NoSuchWindow', - 24 => 'InvalidCookieDomain', - 25 => 'UnableToSetCookie', - 26 => 'UnexpectedAlertOpen', - 27 => 'NoAlertOpenError', - 28 => 'ScriptTimeout', - 29 => 'InvalidElementCoordinates', - 30 => 'IMENotAvailable', - 31 => 'IMEEngineActivationFailed', - 32 => 'InvalidSelector', - 33 => 'SessionNotCreatedException', - 34 => 'MoveTargetOutOfBounds', - ); - - protected static $messages = array( - 6 => 'A session is either terminated or not started', - 7 => 'An element could not be located on the page using the given search parameters.', - 8 => 'A request to switch to a frame could not be satisfied because the frame could not be found.', - 9 => 'The requested resource could not be found, or a request was received using an HTTP method that is not supported by the mapped resource.', - 10 => 'An element command failed because the referenced element is no longer attached to the DOM.', - 11 => 'An element command could not be completed because the element is not visible on the page.', - 12 => 'An element command could not be completed because the element is in an invalid state (e.g. attempting to click a disabled element).', - 13 => 'An unknown server-side error occurred while processing the command.', - 15 => 'An attempt was made to select an element that cannot be selected.', - 17 => 'An error occurred while executing user supplied JavaScript.', - 19 => 'An error occurred while searching for an element by XPath.', - 21 => 'An operation did not complete before its timeout expired.', - 23 => 'A request to switch to a different window could not be satisfied because the window could not be found.', - 24 => 'An illegal attempt was made to set a cookie under a different domain than the current page.', - 25 => 'A request to set a cookie\'s value could not be satisfied.', - 26 => 'A modal dialog was open, blocking this operation', - 27 => 'An attempt was made to operate on a modal dialog when one was not open.', - 28 => 'A script did not complete before its timeout expired.', - 29 => 'The coordinates provided to an interactions operation are invalid.', - 30 => 'IME was not available.', - 31 => 'An IME engine could not be started.', - 32 => 'Argument was an invalid selector (e.g. XPath/CSS).', - 33 => 'A new session could not be created.', - 34 => 'Target provided for a move action is out of bounds.', - ); - - public $name; - - public function __construct($code, \Exception $previous = NULL) - { - $message = isset(self::$messages[$code]) ? self::$messages[$code] : 'Unknown Error'; - $this->name = isset(self::$names[$code]) ? self::$names[$code] : 'UnknownError'; - - parent::__construct($this->name.': '.$message, array(), $previous); - - $this->code = $code; - } -} diff --git a/tests/tests/Driver/Selenium/ConnectionTest.php b/tests/tests/Driver/Selenium/ConnectionTest.php deleted file mode 100644 index f52287c..0000000 --- a/tests/tests/Driver/Selenium/ConnectionTest.php +++ /dev/null @@ -1,41 +0,0 @@ -assertEquals('test', $connection->server()); - $connection->server('http://localhost:4444/wd/hub/'); - - foreach ($connection->get('sessions') as $session) - { - $connection->delete('session/'.$session['id']); - } - - $connection->start(); - - $this->assertTrue($connection->is_started()); - - $connection = new Driver_Selenium_Connection(); - - $connection->start(); - - $this->assertTrue($connection->is_started()); - - $connection = new Driver_Selenium_Connection(); - - $this->assertCount(1, $connection->get('sessions')); - - // var_dump($connection->get('sessions')); - } -} - diff --git a/tests/tests/Driver/SeleniumTest.php b/tests/tests/Driver/SeleniumTest.php deleted file mode 100644 index c96ec72..0000000 --- a/tests/tests/Driver/SeleniumTest.php +++ /dev/null @@ -1,288 +0,0 @@ -base_url('http://6ca1671dbfe9477b14ce-fabb5009fe9cc97c5f42aa7fac8fcd02.r26.cf3.rackcdn.com'); - - self::$driver->visit('/remote-form.html'); - } - - public function find($xpath) - { - $ids = Attempt::make(function() use ($xpath) { - return Driver_SeleniumTest::$driver->all($xpath); - }); - - return isset($ids[0]) ? $ids[0] : NULL; - } - - public function test_accessors() - { - $tag_name = self::$driver->tag_name($this->find("//select[@id='post_category']")); - $this->assertEquals('select', $tag_name); - - $tag_id = self::$driver->attribute($this->find("//select[@id='post_category']"), 'id'); - $this->assertEquals('post_category', $tag_id); - - $html = self::$driver->html($this->find("//textarea[@id='post_body']")); - $this->assertEquals('', $html); - - $text = self::$driver->text($this->find("//textarea[@id='post_body']")); - $this->assertEquals('Lorem Ipsum', $text); - - $value = self::$driver->value($this->find("//input[@id='post_title']")); - $this->assertEquals('Title 1', $value); - - $value = self::$driver->value($this->find("//textarea[@id='post_body']")); - $this->assertEquals('Lorem Ipsum', $value); - - $value = self::$driver->value($this->find("//select[@id='post_category']")); - $this->assertEquals('hw', $value); - - $value = self::$driver->value($this->find("//select[@id='post_ads']")); - $this->assertEquals(array('text', 'affiliate'), $value); - - $is_selected = self::$driver->is_selected($this->find("//option[@value='sw']")); - $this->assertFalse($is_selected); - - $is_selected = self::$driver->is_selected($this->find("//option[@value='hw']")); - $this->assertTrue($is_selected); - - $is_checked = self::$driver->is_checked($this->find("//input[@value='big']")); - $this->assertFalse($is_checked); - - $is_checked = self::$driver->is_checked($this->find("//input[@value='small']")); - $this->assertTrue($is_checked); - } - - public function test_visible() - { - $value = self::$driver->is_visible($this->find("//select[@id='post_ads']")); - $this->assertTrue($value); - - $value = self::$driver->is_visible($this->find("//div[@id='hidden']")); - $this->assertFalse($value); - - $value = self::$driver->is_visible($this->find("//div[@id='visible']")); - $this->assertTrue($value); - } - - public function test_confirm_and_alert_text() - { - self::$driver->execute(NULL, 'if (confirm("Test Confirm?"))alert("Test Alert");'); - - $this->assertEquals('Test Confirm?', self::$driver->alert_text()); - self::$driver->confirm(FALSE); - try - { - self::$driver->alert_text(); - $this->fail('Should not have a dialog open'); - } - catch (Exception_Selenium $exception) - { - // Normal flow - } - - self::$driver->execute(NULL, 'if (confirm("Test Confirm?"))alert("Test Alert");'); - self::$driver->confirm(TRUE); - $this->assertEquals('Test Alert', self::$driver->alert_text()); - self::$driver->confirm(TRUE); - - try - { - self::$driver->alert_text(); - $this->fail('Should not have a dialog open'); - } - catch (Exception_Selenium $exception) - { - // Normal flow - } - } - - public function test_move_to() - { - $id = $this->find("//button[@id='submit-btn']"); - self::$driver->execute($id, "arguments[0].onmouseover = function(){ alert('test'); }"); - self::$driver->move_to($id, 5, 5); - - $this->assertEquals('test', self::$driver->alert_text()); - self::$driver->confirm(FALSE); - self::$driver->visit('/remote-form.html'); - } - - public function test_all() - { - $this->assertCount(1, self::$driver->all('//textarea')); - - $this->assertCount(0, self::$driver->all('//nonexistant-tag')); - } - - public function test_next_query() - { - self::$driver->next_query(array('next' => 'true')); - - self::$driver->visit('/remote-form.html'); - - $this->assertEquals(self::$driver->base_url().'/remote-form.html?next=true', self::$driver->current_url()); - - self::$driver->visit('/remote-form.html'); - - $this->assertEquals(self::$driver->base_url().'/remote-form.html', self::$driver->current_url()); - } - - public function test_screenshot() - { - self::$driver->screenshot(TESTVIEWS.'screenshot.png'); - - $this->assertFileExists(TESTVIEWS.'screenshot.png'); - - unlink(TESTVIEWS.'screenshot.png'); - } - - public function test_content() - { - $html = self::$driver->content(); - $this->assertContains('Author', $html); - } - - public function test_base_url() - { - $old = self::$driver->base_url(); - - self::$driver->base_url('http://example.com'); - - $this->assertEquals('http://example.com', self::$driver->base_url()); - - self::$driver->base_url($old); - } - - public function test_connection() - { - $connection = new Driver_Selenium_Connection(); - $driver = new Driver_Selenium(); - - $driver->connection($connection); - - $this->assertSame($connection, $driver->connection()); - - $driver = new Driver_Selenium(); - $connection = $driver->connection(); - $this->assertInstanceOf('OpenBuildings\Spiderling\Driver_Selenium_Connection', $connection); - $this->assertTrue($connection->is_started()); - unset($driver); - } - - public function test_cookie() - { - self::$driver->cookie('test', 'value', array('path' => '/')); - - $cookies = self::$driver->cookies(); - - $this->assertEquals('test', $cookies[0]['name']); - $this->assertEquals('value', $cookies[0]['value']); - $this->assertEquals('/', $cookies[0]['path']); - - self::$driver->clear(); - - $this->assertCount(0, self::$driver->cookies()); - } - - public function test_actions() - { - $this->assertValueSet($this->find("//input[@id='post_title']"), 'New Title', 'New Title', self::$driver); - - $this->assertValueSet($this->find("//textarea[@id='post_body']"), 'New Text', 'New Title', self::$driver); - - $this->assertValueSet($this->find("//input[@value='tiny']"), TRUE, 'tiny', self::$driver); - - $this->assertValueSet($this->find("//input[@value='sendval']"), TRUE, 'sendval', self::$driver); - - self::$driver->select_option($this->find("//select[@id='post_category']//option[text()='Software']"), TRUE); - - $value = self::$driver->value($this->find("//select[@id='post_category']")); - $this->assertEquals('sw', $value); - - $old_is_selected = self::$driver->is_selected($this->find("//select[@id='post_category']//option[text()='Hardware']")); - $this->assertFalse($old_is_selected); - - self::$driver->select_option($this->find("//select[@id='post_ads']//option[text()='Banner']"), TRUE); - - $value = self::$driver->value($this->find("//select[@id='post_ads']")); - $this->assertEquals(array('banner', 'text', 'affiliate'), $value); - - self::$driver->select_option($this->find("//select[@id='post_ads']//option[text()='Text']"), FALSE); - - $value = self::$driver->value($this->find("//select[@id='post_ads']")); - $this->assertEquals(array('banner', 'affiliate'), $value); - - $value = self::$driver->value($this->find("//select[@id='post_ads']")); - - self::$driver->set($this->find("//select[@id='post_ads']//option[text()='Text']"), TRUE); - - $value = self::$driver->value($this->find("//select[@id='post_ads']")); - $this->assertEquals(array('banner', 'text', 'affiliate'), $value); - - self::$driver->set($this->find("//select[@id='post_ads']//option[text()='Text']"), FALSE); - - $value = self::$driver->value($this->find("//select[@id='post_ads']")); - $this->assertEquals(array('banner', 'affiliate'), $value); - - self::$driver->set($this->find("//input[@type='file']"), TESTVIEWS.'form.html'); - - $value = self::$driver->value($this->find("//input[@type='file']")); - - $this->assertEquals('form.html', $value); - } - - public function test_clicks() - { - self::$driver->click($this->find("//a[@id='visible-link']")); - - $title = self::$driver->text($this->find("//h1")); - $this->assertEquals('Linked', $title); - - $this->assertEquals(self::$driver->base_url().'/remote-linked.html', self::$driver->current_url()); - - $this->assertEquals('/remote-linked.html', self::$driver->current_path()); - - self::$driver->visit('/remote-form.html'); - - self::$driver->click($this->find("//button[@id='submit-btn']")); - $title = self::$driver->text($this->find("//h1")); - $this->assertEquals('Submitted', $title); - - self::$driver->visit('/remote-form.html'); - - self::$driver->click($this->find("//input[@id='submit']")); - $title = self::$driver->text($this->find("//h1")); - $this->assertEquals('Submitted', $title); - - self::$driver->visit('/remote-form.html'); - } - - public function test_is_page_active() - { - $this->assertTrue(self::$driver->is_page_active()); - - $driver = new Driver_Selenium(); - - $this->assertFalse($driver->is_page_active()); - } -} - From a3bc3bcd51425faaab75a199ad8022977ef3a953 Mon Sep 17 00:00:00 2001 From: Nikolay Tumbalev Date: Thu, 13 Feb 2020 12:35:57 +0200 Subject: [PATCH 4/7] Update the repo readme. --- README.md | 40 +++------------------------------------- 1 file changed, 3 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index e9f5f44..5ec6f7c 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![Code Coverage](https://scrutinizer-ci.com/g/OpenBuildings/spiderling/badges/coverage.png?s=f056fa81f6a4f1fde71505682083b8f75b42d9c0)](https://scrutinizer-ci.com/g/OpenBuildings/spiderling/) [![Latest Stable Version](https://poser.pugx.org/openbuildings/spiderling/v/stable.png)](https://packagist.org/packages/openbuildings/spiderling) -This is a library for crawling web pages with curl, PhantomJS and Selenium. Heavily inspired by [Capybara](https://github.com/jnicklas/capybara). It's a major component in [phpunit-spiderling](https://github.com/OpenBuildings/phpunit-spiderling) for integration level testing. It can handle AJAX requests easily and allows switching from fast PHP-only drivers to JavaScript-enabled ones like PhantomJS and Selenium easily, without modifying the code. +This is a library for crawling web pages with curl and PhantomJS. Heavily inspired by [Capybara](https://github.com/jnicklas/capybara). It's a major component in [phpunit-spiderling](https://github.com/OpenBuildings/phpunit-spiderling) for integration level testing. It can handle AJAX requests easily and allows switching from fast PHP-only drivers to JavaScript-enabled like PhantomJS easily, without modifying the code. ## A quick example @@ -48,7 +48,7 @@ Each node represents a HTML tag on the page, and you can use extensive getter me - ``tag_name()``: Get the tag name of the dom element. e.g. DIV, SPAN, FORM, SELECT - ``attribute($name)``: Get an attribute of the current tag. If the tag is empty e.g. ``
`` then it will return an empty string. If there is no attribute however, NULL will be returned - ``text()``: Get the text content of an html tag - this is similar to how browsers render HTML tags, all whitespace will be merged to single spaces. -- ``is_visible()``: Check if a node is visible. PhantomJS and Selenium drivers will return correct value if the item is hidden via JS, CSS or inline styles. +- ``is_visible()``: Check if a node is visible. PhantomJS driver will return correct value if the item is hidden via JS, CSS or inline styles. - ``is_selected()``: Check if an option tag is "selected" - ``is_checked()``: Check if an input tag is "checked" - ``value()``: Get the value of an input form tag @@ -494,7 +494,7 @@ $page ## Drivers -A great strength of Spiderling is the ability to use different drivers for your code. This allows switching from PHP-only curl parsing of the page to a PhantomJS or even Selenium without modification of the code. For example if we wanted to use a PhantomJS driver instead of the default "Simple" one then we'd need to do this: +A great strength of Spiderling is the ability to use different drivers for your code. This allows switching from PHP-only curl parsing of the page to a PhantomJS without modification of the code. For example if we wanted to use a PhantomJS driver instead of the default "Simple" one then we'd need to do this: ```php use Openbuildings\Spiderling\Page; @@ -516,7 +516,6 @@ There are 4 drivers at present: - __Driver_Simple__: Uses PHP curl to load pages. Does not support JavaScript or browser alert dialogs - __Driver_Kohana__: Uses Kohana framework's native Internal Request class, without opening internet connections at all - very performant if your code already uses Kohana framework. - __Driver_Phantomjs__: Start a PhantomJS server. You would need to have PhantomJS installed and accessible in your PATH. Picks a new port at random so its possible to have multiple PhantomJS browsers open simultaneously. -- __Driver_Selenium__: Uses Selenium server. Reuses a session if possible, you have to start the server independantly. You can easily write your own Drivers by extending the Driver class and implementing methods yourself. Some drivers do not support all the features, so it's OK to not implement every method. @@ -605,39 +604,6 @@ $page = new Page(); Setting the "pid file" argument on start, allows the driver to save the pid of the phantomjs server process to that file, and then try to clean up the server when started again, thus making sure you don't have running PhantomJS process all over the place. - -### Driver_Selenium - -Using this driver all the finds and actions are performed through a real browser, driven by Selenium. - -You can use the standalone server for example, which can be downloaded from here: https://code.google.com/p/selenium/downloads/list. - -You'll also have to start the Selenium server yourself, and direct the driver what URL to use to access that server, by default its "http://localhost:4444" (the default for Selenium server standalone). - -If you're using the standalone Selenium, you can start it like this: - -``` -cd /{where the Selenium server standalone jar is} -java -jar selenium-server-standalone-2.*.jar -``` - -After it's started you can access it like this - -```php -use Openbuildings\Spiderling\Page; - -$page = new Page(new Driver_Selenium); -``` - -or if you have it running somewhere else (a cluster or on a different host altogether) - -```php -use Openbuildings\Spiderling\Page; - -$connection = new Driver_Selenium_Connection('http://server.example.com:4444/wc/hub'); -$page = new Page(new Driver_Selenium($connection)); -``` - ## License Copyright (c) 2012-2013, OpenBuildings Ltd. Developed by Ivan Kerin as part of [clippings.com](http://clippings.com) From db4d127eb6a4c4107302ad110104b520cee59d0e Mon Sep 17 00:00:00 2001 From: Nikolay Tumbalev Date: Thu, 13 Feb 2020 13:39:57 +0200 Subject: [PATCH 5/7] Remove older versions than php 5.6 --- .travis.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7dfbc3e..8f40619 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,9 +3,6 @@ language: php sudo: false php: - - 5.3 - - 5.4 - - 5.5 - 5.6 - 7.0 - 7.1 @@ -15,14 +12,6 @@ env: - DEPS=highest - DEPS=lowest -matrix: - fast_finish: true - exclude: - - php: 5.3 - env: DEPS=highest - - php: 5.4 - env: DEPS=highest - branches: only: - master From 2e9083f5f8974b590cd824cea3019b4167847832 Mon Sep 17 00:00:00 2001 From: Nikolay Tumbalev Date: Thu, 13 Feb 2020 14:41:25 +0200 Subject: [PATCH 6/7] Extract xvfb as service. --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8f40619..d4525a6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,8 @@ language: php sudo: false +services: xvfb + php: - 5.6 - 7.0 @@ -45,7 +47,6 @@ install: before_script: - mkdir -p build/logs - "export DISPLAY=:99.0" - - "sh -e /etc/init.d/xvfb start" - sleep 3 script: From ed0fb1a8850b391a118b46c408334f5540eb5ecd Mon Sep 17 00:00:00 2001 From: Nikolay Tumbalev Date: Fri, 14 Feb 2020 12:23:09 +0200 Subject: [PATCH 7/7] Change the old cdn link with new one. --- tests/tests/Driver/PhantomjsTest.php | 3 +-- tests/tests/Driver/Simple/RequestFactory/HTTPTest.php | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/tests/Driver/PhantomjsTest.php b/tests/tests/Driver/PhantomjsTest.php index c1af0d3..6bbb0a2 100644 --- a/tests/tests/Driver/PhantomjsTest.php +++ b/tests/tests/Driver/PhantomjsTest.php @@ -18,8 +18,7 @@ public static function setUpBeforeClass() parent::setUpBeforeClass(); self::$driver = new Driver_Phantomjs(); - self::$driver->base_url('https://85b5d31b11244c8d6302-fabb5009fe9cc97c5f42aa7fac8fcd02.ssl.cf3.rackcdn.com'); - + self::$driver->base_url('http://clippings-spiderling.s3-website-eu-west-1.amazonaws.com'); self::$driver->visit('/remote-form.html'); } diff --git a/tests/tests/Driver/Simple/RequestFactory/HTTPTest.php b/tests/tests/Driver/Simple/RequestFactory/HTTPTest.php index 5155057..53fd939 100644 --- a/tests/tests/Driver/Simple/RequestFactory/HTTPTest.php +++ b/tests/tests/Driver/Simple/RequestFactory/HTTPTest.php @@ -18,15 +18,15 @@ public function setUp() public function test_request() { - $content = $this->factory->execute('GET', 'http://6ca1671dbfe9477b14ce-fabb5009fe9cc97c5f42aa7fac8fcd02.r26.cf3.rackcdn.com/remote-form.html', array('test' => 'value')); + $content = $this->factory->execute('GET', 'http://clippings-spiderling.s3-website-eu-west-1.amazonaws.com/remote-form.html', array('test' => 'value')); $this->assertContains('Author', $content); - $this->assertEquals('http://6ca1671dbfe9477b14ce-fabb5009fe9cc97c5f42aa7fac8fcd02.r26.cf3.rackcdn.com/remote-form.html', $this->factory->current_url()); + $this->assertEquals('http://clippings-spiderling.s3-website-eu-west-1.amazonaws.com/remote-form.html', $this->factory->current_url()); $this->assertEquals('/remote-form.html', $this->factory->current_path()); $this->setExpectedException('Openbuildings\Spiderling\Exception_Curl'); - $this->factory->execute('GET', 'http://6ca1671dbfe9477b14ce-fabb5009fe9cc97c5f42aa7fac8fcd02.r26.cf3.rackcdn.com/not-existst.html'); + $this->factory->execute('GET', 'http://clippings-spiderling.s3-website-eu-west-1.amazonaws.com/not-existst.html'); }