Feature/updating indeed scraper (#166) #34
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This tests all JobFunnel Scrapes | |
| name: JobFunnel CI | |
| on: | |
| push: | |
| branches: [master] | |
| pull_request: | |
| branches: [master] | |
| jobs: | |
| build: | |
| runs-on: ubuntu-16.04 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up Python 3.8 | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: 3.8 | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| - name: Install JobFunnel | |
| run: | | |
| pip install -e . | |
| python -m nltk.downloader stopwords | |
| - name: Lint with flake8 | |
| run: | | |
| # stop the build if there are Python syntax errors or undefined names | |
| flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics | |
| - name: Test with pytest | |
| run: | | |
| pytest | |
| - name: Run CANADA_ENGLISH demo by settings YAML | |
| run: | | |
| funnel load -s demo/settings.yaml -log-level DEBUG | |
| - name: Run an american search by CLI | |
| run: | | |
| funnel inline -kw Python Data Scientist PHD AI -ps WA -c Seattle -l USA_ENGLISH -log-level DEBUG -csv demo_job_search_results/demo_search.csv -cache demo_job_search_results/cache2 -blf demo_job_search_results/demo_block_list.json -dl demo_job_search_results/demo_duplicates_list.json -log-file demo_job_search_results/log.log -max-listing-days 1 | |
| - name: Run a FRANCE_FRENCH demo by settings YAML | |
| run: | | |
| funnel load -s demo/settings_FR.yaml -log-level DEBUG | |
| - name: Run a GERMANY_GERMAN demo by settings YAML | |
| run: | | |
| funnel load -s demo/settings_DE.yaml -log-level DEBUG | |
| - name: Obtain coverage | |
| run: | | |
| pytest --cov=jobfunnel --cov-report=xml | |
| - name: After Success | |
| run: | | |
| bash <(curl -s https://codecov.io/bash) | |
| - name: Run a Remote jobs only scrape | |
| run: | | |
| funnel inline -kw Python -ps ON -c Toronto -l CANADA_ENGLISH -remoteness FULLY_REMOTE -p INDEED -log-level DEBUG -csv demo_job_search_results/demo_remote_search.csv -cache demo_job_search_results/cache3 -blf demo_job_search_results/demo_block_list.json -dl demo_job_search_results/demo_duplicates_list.json -log-file demo_job_search_results/log.log -max-listing-days 3 | |
| # TODO: modify some job statuses and run with --no-scrape... | |
| # - './tests/verify_time.sh' TODO: some way of verifying execution time | |
| # - './demo/gen_call_graphs.sh' TODO: some way of showing .dot on GitHub? |