-
Notifications
You must be signed in to change notification settings - Fork 0
/
.travis.yml
165 lines (136 loc) · 6.09 KB
/
.travis.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
sudo: false
language: python
env:
global:
- DBNAME=era_data
matrix:
include:
- env: DB=pgsql
python: 2.7
virtualenv:
system_site_packages: true
addons:
apt:
packages:
- python-lxml
- python-psycopg2 # python3-psycopg2 is also available
- python-sqlalchemy
- env: DB=oracle
language: python
python: 3.4
sudo: required
cache: false
- env: DB=sqlite3
python: 2.7
virtualenv:
system_site_packages: true
addons:
apt:
packages:
- python-lxml
- python-sqlalchemy
- env: DB=mysql
python: 2.7
virtualenv:
system_site_packages: true
addons:
apt:
packages:
- python-lxml
- python-mysqldb # No Python 3 support
- python-sqlalchemy
before_install:
# 'sqlite3' and 'mysql' are the command name, and they are quiet by default
- SQLCMD=$DB
- SQLCMD_QUIET=$SQLCMD
- if [ "$DB" = 'sqlite3' ]; then
DBNAME="$DBNAME.sqlite" ;
DBURL="sqlite:///era_data.sqlite" ;
PYDBD="" ;
fi
- if [ "$DB" = 'pgsql' ]; then
export PGUSER=postgres ;
DBURL="postgresql://postgres@localhost/era_data" ;
PYDBD=psycopg2 ;
SQLCMD=psql ;
SQLCMD_QUIET="psql -q" ;
fi
# pip install mysqldb fails 'Could not find any downloads that satisfy the requirement mysqldb'
- if [ "$DB" = 'mysql' ]; then
DBURL="mysql://travis@localhost/era_data?charset=utf8" ;
PYDBD="" ;
fi
- if [ "$DB" = 'oracle' ]; then
export DBNAME="/" ;
DBURL="oracle://" ;
PYDBD=cx_oracle ;
if [ -z "$ORACLE_LOGIN_ssousername" -o -z "$ORACLE_LOGIN_password" ]; then
echo ORACLE_LOGIN_ssousername or ORACLE_LOGIN_password are not set ;
echo Oracle XE can not be installed ;
exit 101 ;
fi ;
SQLCMD=sqlplus ;
SQLCMD_QUIET="sqlplus -s" ;
export ORACLE_COOKIE=sqldev ;
export ORACLE_FILE=oracle11g/xe/oracle-xe-11.2.0-1.0.x86_64.rpm.zip ;
export ORACLE_HOME=/u01/app/oracle/product/11.2.0/xe ;
export ORACLE_SID=XE ;
export PATH=$PATH:$ORACLE_HOME/bin ;
export LD_LIBRARY_PATH=$ORACLE_HOME/lib ;
export NLS_LANG=American_America.UTF8 ;
wget 'https://github.com/cbandy/travis-oracle/archive/v2.0.0.tar.gz'; mkdir -p .travis/oracle ; tar xz --strip-components 1 -C .travis/oracle -f v2.0.0.tar.gz ;
bash -x .travis/oracle/download.sh ;
bash -x .travis/oracle/install.sh ;
fi
# cx_Oracle workaround https://bitbucket.org/zzzeek/sqlalchemy/issues/3705
- if [ "$DB" = 'oracle' ]; then pip install "sqlalchemy>=1.0.13" ; fi
- export SQLCMD; export SQLCMD_QUIET
before_script:
# Create schema scripts with tables prefixed era201x
- sed -re 's/(^CREATE TABLE|REFERENCES|CONSTRAINT) journal([_ ])/\1 era2010_journal\2/g' schema.sql > schema2010.sql
- sed -re 's/(^CREATE TABLE|REFERENCES|CONSTRAINT) journal([_ ])/\1 era2012_journal\2/g' schema.sql > schema2012.sql
- sed -re 's/(^CREATE TABLE|REFERENCES|CONSTRAINT) journal([_ ])/\1 era2015_journal\2/g' schema.sql > schema2015.sql
- sed -re 's/(^CREATE TABLE|REFERENCES|CONSTRAINT) journal([_ ])/\1 era2015c_journal\2/g' schema.sql > schema2015_consultation.sql
- sed -re 's/(^CREATE TABLE|REFERENCES|CONSTRAINT) journal([_ ])/\1 era2015s_journal\2/g' schema.sql > schema2015_submitted.sql
- if [ "$DB" = 'mysql' ]; then sed -ri 's/n?varchar2\((1?[0-9][0-9]?)\)/varchar(\1)/;s/n?varchar2\(([0-9]+)\)/text(\1)/' schema201*.sql ; fi
- if [ "$DB" = 'pgsql' ]; then sed -ri 's/n?varchar2\(([0-9]+)\)/varchar(\1)/' schema201*.sql ; fi
- if [ "$DB" != 'sqlite3' -a "$DB" != 'oracle' ]; then echo 'create database era_data;' | $SQLCMD; fi
# Populate database with both schema
- $SQLCMD_QUIET $DBNAME < schema2010.sql
- $SQLCMD_QUIET $DBNAME < schema2012.sql
- $SQLCMD_QUIET $DBNAME < schema2015.sql
- $SQLCMD_QUIET $DBNAME < schema2015_consultation.sql
- $SQLCMD_QUIET $DBNAME < schema2015_submitted.sql
script:
# Download the journal lists and convert them to xml
- sh -x fetch.sh
- ls -l *.xml
# Remove the duplicate disciplines from the ERA2010 journal list
- mv ERA2010_journal_title_list.xml ERA2010_journal_title_list.xml.orig
- lxmlproc --output ERA2010_journal_title_list.xml era_journal_list_2010_tidy.xsl ERA2010_journal_title_list.xml.orig
- diff -uw ERA2010_journal_title_list.xml.orig ERA2010_journal_title_list.xml || true # || true ignores the exit code from diff
# Remove the duplicate disciplines from the ERA2015 journal list
- mv ERA2015_journal_title_list.xml ERA2015_journal_title_list.xml.orig
- lxmlproc --output ERA2015_journal_title_list.xml era_journal_list_2010_tidy.xsl ERA2015_journal_title_list.xml.orig
- diff -uw ERA2015_journal_title_list.xml.orig ERA2015_journal_title_list.xml || true # || true ignores the exit code from diff
# Convert the xml to sql insert statements
- perl era_journal_list_sql.pl ERA2010_journal_title_list.xml era2010 $DB > ERA2010_journal_title_list.sql
- perl era_journal_list_sql.pl ERA2012_journal_title_list.xml era2012 $DB > ERA2012_journal_title_list.sql
- perl era_journal_list_sql.pl ERA2015_journal_title_list.xml era2015 $DB > ERA2015_journal_title_list.sql
- perl era_journal_list_sql.pl ERA2015_consultation_journal_title_list.xml era2015c $DB > ERA2015_consultation_journal_title_list.sql
- perl era_journal_list_sql.pl ERA2015_submitted_journal_title_list.xml era2015s $DB > ERA2015_submitted_journal_title_list.sql
- ls -l *.sql
# Insert the data into database
- $SQLCMD_QUIET $DBNAME < ERA2010_journal_title_list.sql
- $SQLCMD_QUIET $DBNAME < ERA2012_journal_title_list.sql
- $SQLCMD_QUIET $DBNAME < ERA2015_journal_title_list.sql
- $SQLCMD_QUIET $DBNAME < ERA2015_consultation_journal_title_list.sql
- $SQLCMD_QUIET $DBNAME < ERA2015_submitted_journal_title_list.sql
- if [ "$DB" = 'sqlite3' ]; then ls -al era_data.sqlite*; fi
- if [ -n "$PYDBD" ]; then pip install $PYDBD; fi
- py.test --sqlalchemy-connect-url="$DBURL"
notifications:
email: false
cache:
directories:
- .cache/era_xml