Merge branch 'release/19.29.0'

CHANGELOG

@@ -2,6 +2,12 @@
 
 We follow the CalVer (https://calver.org/) versioning scheme: YY.MINOR.MICRO.
 
+19.29.0 (2019-10-02)
+===================
+- Use new pagecounter fields for increased query efficiency
+- Allow meetings to be sorted on download count
+- Remove old permissions fields now that we have Guardian
+
 19.28.0 (2019-09-24)
 ===================
 - API v2: Use consistent naming for JSON API type (kebab-case)

Dockerfile

@@ -1,99 +1,27 @@
-FROM python:2.7-slim-stretch
-
-ENV GOSU_VERSION=1.10 \
- NODE_ENV=production \
- NODE_VERSION=8.6.0 \
- YARN_VERSION=1.1.0
-
-# Libraries such as matplotlib require a HOME directory for cache and configuration
-RUN set -ex \
- && mkdir -p /var/www \
- && chown www-data:www-data /var/www \
- && apt-get update \
- && apt-get install -y gnupg2 \
- && for key in \
- # GOSU
- B42F6819007F00F88E364FD4036A9C25BF357DD4 \
- # https://github.com/nodejs/docker-node/blob/9c25cbe93f9108fd1e506d14228afe4a3d04108f/8.2/Dockerfile
- # gpg keys listed at https://github.com/nodejs/node#release-team
- # Node
- 9554F04D7259F04124DE6B476D5A82AC7E37093B \
- 94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \
- FD3A5288F042B6850C66B31F09FE44734EB7990E \
- 71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 \
- DD8F2338BAE7501E3DD5AC78C273792F7D83545D \
- B9AE9905FFD7803F25714661B63B535A4C206CA9 \
- C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 \
- 56730D5401028683275BD23C23EFEFE93C4CFFFE \
- # Yarn
- 6A010C5166006599AA17F08146C2130DFD2497F5 \
- ; do \
- gpg --keyserver hkp://ipv4.pool.sks-keyservers.net:80 --recv-keys "$key" || \
- gpg --keyserver hkp://ha.pool.sks-keyservers.net:80 --recv-keys "$key" || \
- gpg --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" || \
- gpg --keyserver hkp://keyserver.pgp.com:80 --recv-keys "$key" \
- ; done \
- # Install dependancies
- && apt-get install -y \
- git \
- libev4 \
- libev-dev \
- libevent-dev \
- libxml2-dev \
- libxslt1-dev \
- zlib1g-dev \
- curl \
- # cryptography
- build-essential \
- libssl-dev \
- libffi-dev \
- python-dev \
- # postgresql
- libpq-dev \
- # file audits
- par2 \
- && ARCH= \
- && dpkgArch="$(dpkg --print-architecture)" \
- && case "${dpkgArch##*-}" in \
- amd64) ARCH='x64';; \
- ppc64el) ARCH='ppc64le';; \
- *) echo "unsupported architecture"; exit 1 ;; \
- esac \
- # grab gosu for easy step-down from root
- && curl -o /usr/local/bin/gosu -SL "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch" \
- && curl -o /usr/local/bin/gosu.asc -SL "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch.asc" \
- && gpg --verify /usr/local/bin/gosu.asc \
- && rm /usr/local/bin/gosu.asc \
- && chmod +x /usr/local/bin/gosu \
- # Node
- && curl -SLO "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH.tar.xz" \
- && curl -SLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \
- && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc \
- && grep " node-v$NODE_VERSION-linux-$ARCH.tar.xz\$" SHASUMS256.txt | sha256sum -c - \
- && tar -xJf "node-v$NODE_VERSION-linux-$ARCH.tar.xz" -C /usr/local --strip-components=1 \
- && rm "node-v$NODE_VERSION-linux-$ARCH.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt \
- && ln -s /usr/local/bin/node /usr/local/bin/nodejs \
- # Yarn
- && curl -fSLO --compressed "https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz" \
- && curl -fSLO --compressed "https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz.asc" \
- && gpg --batch --verify yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz \
- && mkdir -p /opt/yarn \
- && tar -xzf yarn-v$YARN_VERSION.tar.gz -C /opt/yarn --strip-components=1 \
- && ln -s /opt/yarn/bin/yarn /usr/local/bin/yarn \
- && ln -s /opt/yarn/bin/yarn /usr/local/bin/yarnpkg \
- && yarn global add bower \
- && yarn cache clean \
- && rm -rf \
- yarn-v$YARN_VERSION.tar.gz.asc \
- yarn-v$YARN_VERSION.tar.gz \
- $HOME/.gnupg \
- $HOME/.cache \
- && apt-get remove -y \
- curl \
- && apt-get clean \
- && apt-get autoremove -y \
- && rm -rf /var/lib/apt/lists/* \
- && mkdir -p /code
+FROM node:8-alpine
+
+# Source: https://github.com/docker-library/httpd/blob/7976cabe162268bd5ad2d233d61e340447bfc371/2.4/alpine/Dockerfile#L3
+RUN set -x \
+ && addgroup -g 82 -S www-data \
+ && adduser -h /var/www -u 82 -D -S -G www-data www-data
+
+RUN apk add --no-cache --virtual .run-deps \
+ su-exec \
+ bash \
+ python \
+ py-pip \
+ git \
+ # lxml2
+ libxml2 \
+ libxslt \
+ # psycopg2
+ postgresql-libs \
+ # cryptography
+ libffi \
+ # gevent
+ libev \
+ libevent \
+ && yarn global add bower
 
 WORKDIR /code
 
@@ -118,7 +46,22 @@ COPY ./addons/twofactor/requirements.txt ./addons/twofactor/
 #COPY ./addons/wiki/requirements.txt ./addons/wiki/
 COPY ./addons/zotero/requirements.txt ./addons/zotero/
 
-RUN for reqs_file in \
+RUN set -ex \
+ && mkdir -p /var/www \
+ && chown www-data:www-data /var/www \
+ && apk add --no-cache --virtual .build-deps \
+ build-base \
+ linux-headers \
+ python-dev \
+ # lxml2
+ musl-dev \
+ libxml2-dev \
+ libxslt-dev \
+ # psycopg2
+ postgresql-dev \
+ # cryptography
+ libffi-dev \
+ && for reqs_file in \
  /code/requirements.txt \
  /code/requirements/release.txt \
  /code/addons/*/requirements.txt \
@@ -128,17 +71,10 @@ RUN for reqs_file in \
  && (pip uninstall uritemplate.py --yes || true) \
  && pip install --no-cache-dir uritemplate.py==0.3.0 \
  # Fix: https://github.com/CenterForOpenScience/osf.io/pull/6783
- && python -m compileall /usr/local/lib/python2.7 || true
-
-# OSF: Assets
-COPY ./.bowerrc ./bower.json ./
-RUN bower install --production --allow-root \
- && bower cache clean --allow-root
-
-COPY ./package.json ./.yarnrc ./yarn.lock ./
-RUN yarn install --frozen-lockfile \
- && yarn cache clean
+ && python -m compileall /usr/lib/python2.7 || true \
+ && apk del .build-deps
 
+# Settings
 COPY ./tasks/ ./tasks/
 COPY ./website/settings/ ./website/settings/
 COPY ./api/base/settings/ ./api/base/settings/
@@ -148,8 +84,29 @@ RUN mv ./website/settings/local-dist.py ./website/settings/local.py \
  && mv ./api/base/settings/local-dist.py ./api/base/settings/local.py \
  && sed 's/DEBUG_MODE = True/DEBUG_MODE = False/' -i ./website/settings/local.py
 
+# Bower Assets
+COPY ./.bowerrc ./bower.json ./
+COPY ./admin/.bowerrc ./admin/bower.json ./admin/
+RUN \
+ # OSF
+ bower install --production --allow-root \
+ && bower cache clean --allow-root \
+ # Admin
+ && cd ./admin \
+ && bower install --production --allow-root \
+ && bower cache clean --allow-root
+
+# Webpack Assets
+#
+## OSF
+COPY ./package.json ./.yarnrc ./yarn.lock ./
 COPY ./webpack* ./
 COPY ./website/static/ ./website/static/
+## Admin
+COPY ./admin/package.json ./admin/yarn.lock ./admin/
+COPY ./admin/webpack* ./admin/
+COPY ./admin/static/ ./admin/static/
+## Addons
 COPY ./addons/bitbucket/static/ ./addons/bitbucket/static/
 COPY ./addons/box/static/ ./addons/box/static/
 COPY ./addons/citations/static/ ./addons/citations/static/
@@ -168,34 +125,29 @@ COPY ./addons/s3/static/ ./addons/s3/static/
 COPY ./addons/twofactor/static/ ./addons/twofactor/static/
 COPY ./addons/wiki/static/ ./addons/wiki/static/
 COPY ./addons/zotero/static/ ./addons/zotero/static/
-RUN mkdir -p ./website/static/built/ \
+RUN \
+ # OSF
+ yarn install --frozen-lockfile \
+ && mkdir -p ./website/static/built/ \
  && invoke build_js_config_files \
- && yarn run webpack-prod
-# /OSF: Assets
-
-# Admin: Assets
-COPY ./admin/.bowerrc ./admin/bower.json ./admin/
-RUN cd ./admin \
- && bower install --production --allow-root \
- && bower cache clean --allow-root
-
-COPY ./admin/package.json ./admin/yarn.lock ./admin/
-RUN cd ./admin \
+ && yarn run webpack-prod \
+ # Admin
+ && cd ./admin \
  && yarn install --frozen-lockfile \
- && yarn cache clean
-
-COPY ./admin/webpack* ./admin/
-COPY ./admin/static/ ./admin/static/
-RUN cd ./admin \
- && yarn run webpack-prod
-# /Admin: Assets
+ && yarn run webpack-prod \
+ && cd ../ \
+ # Cleanup
+ && yarn cache clean \
+ && npm cache clean --force
 
 # Copy the rest of the code over
 COPY ./ ./
 
 ARG GIT_COMMIT=
 ENV GIT_COMMIT ${GIT_COMMIT}
 
+# TODO: Admin/API should fully specify their bower static deps, and not include ./website/static in their defaults.py.
+# (this adds an additional 300+mb to the build image)
 RUN for module in \
  api.base.settings \
  admin.base.settings \
@@ -211,4 +163,4 @@ RUN for module in \
  ; done \
  && rm ./website/settings/local.py ./api/base/settings/local.py
 
-CMD ["gosu", "nobody", "invoke", "--list"]
+CMD ["su-exec", "nobody", "invoke", "--list"]

addons/base/views.py

@@ -330,9 +330,9 @@ def get_auth(auth, **kwargs):
  # version index is 0 based
  version_index = version - 1
  if action == 'render':
- update_analytics(node, file_id, version_index, 'view')
+ update_analytics(node, filenode, version_index, 'view')
  elif action == 'download' and not from_mfr:
- update_analytics(node, file_id, version_index, 'download')
+ update_analytics(node, filenode, version_index, 'download')
  if waffle.switch_is_active(features.ELASTICSEARCH_METRICS):
  if isinstance(node, Preprint):
  metric_class = get_metric_class_for_action(action, from_mfr=from_mfr)
@@ -517,7 +517,6 @@ def create_waterbutler_log(payload, **kwargs):
 def addon_delete_file_node(self, target, user, event_type, payload):
  """ Get addon BaseFileNode(s), move it into the TrashedFileNode collection
  and remove it from StoredFileNode.
-
  Required so that the guids of deleted addon files are not re-pointed when an
  addon file or folder is moved or renamed.
  """

addons/osfstorage/tests/test_models.py

@@ -184,9 +184,9 @@ def test_download_count_file(self, mock_session):
  mock_session.data = {}
  child = self.node_settings.get_root().append_file('Test')
 
- utils.update_analytics(self.project, child._id, 0)
- utils.update_analytics(self.project, child._id, 1)
- utils.update_analytics(self.project, child._id, 2)
+ utils.update_analytics(self.project, child, 0)
+ utils.update_analytics(self.project, child, 1)
+ utils.update_analytics(self.project, child, 2)
 
  assert_equals(child.get_download_count(), 3)
  assert_equals(child.get_download_count(0), 1)

addons/osfstorage/tests/test_utils.py

@@ -31,9 +31,9 @@ def setUp(self):
 
  def test_serialize_revision(self):
  sessions.sessions[request._get_current_object()] = Session()
- utils.update_analytics(self.project, self.record._id, 0)
- utils.update_analytics(self.project, self.record._id, 0)
- utils.update_analytics(self.project, self.record._id, 2)
+ utils.update_analytics(self.project, self.record, 0)
+ utils.update_analytics(self.project, self.record, 0)
+ utils.update_analytics(self.project, self.record, 2)
  expected = {
  'index': 1,
  'user': {
@@ -58,9 +58,9 @@ def test_serialize_revision(self):
 
  def test_anon_revisions(self):
  sessions.sessions[request._get_current_object()] = Session()
- utils.update_analytics(self.project, self.record._id, 0)
- utils.update_analytics(self.project, self.record._id, 0)
- utils.update_analytics(self.project, self.record._id, 2)
+ utils.update_analytics(self.project, self.record, 0)
+ utils.update_analytics(self.project, self.record, 0)
+ utils.update_analytics(self.project, self.record, 2)
  expected = {
  'index': 2,
  'user': None,

addons/osfstorage/utils.py

@@ -16,24 +16,28 @@
 LOCATION_KEYS = ['service', settings.WATERBUTLER_RESOURCE, 'object']
 
 
-def update_analytics(node, file_id, version_idx, action='download'):
+def update_analytics(node, file, version_idx, action='download'):
  """
  :param Node node: Root node to update
  :param str file_id: The _id field of a filenode
  :param int version_idx: Zero-based version index
  :param str action: is this logged as download or a view
  """
- # Pass in contributors to check that contributors' downloads
+ # Pass in contributors and group members to check that their downloads
  # do not count towards total download count
  contributors = []
- if node.contributors:
+ if getattr(node, 'contributors_and_group_members', None):
+ contributors = node.contributors_and_group_members
+ elif getattr(node, 'contributors', None):
  contributors = node.contributors
+
  node_info = {
  'contributors': contributors
  }
+ resource = node.guids.first()
 
- update_counter('{0}:{1}:{2}'.format(action, node._id, file_id), node_info=node_info)
- update_counter('{0}:{1}:{2}:{3}'.format(action, node._id, file_id, version_idx), node_info=node_info)
+ update_counter(resource, file, version=None, action=action, node_info=node_info)
+ update_counter(resource, file, version_idx, action, node_info=node_info)
 
 
 def serialize_revision(node, record, version, index, anon=False):
@@ -44,7 +48,6 @@ def serialize_revision(node, record, version, index, anon=False):
  :param FileVersion version: The version to serialize
  :param int index: One-based index of version
  """
-
  if anon:
  user = None
  else:

addons/osfstorage/views.py

@@ -106,8 +106,7 @@ def osfstorage_get_revisions(file_node, payload, target, **kwargs):
  counter_prefix = 'download:{}:{}:'.format(file_node.target._id, file_node._id)
 
  version_count = file_node.versions.count()
- # Don't worry. The only % at the end of the LIKE clause, the index is still used
- counts = dict(PageCounter.objects.filter(_id__startswith=counter_prefix).values_list('_id', 'total'))
+ counts = dict(PageCounter.objects.filter(resource=file_node.target.guids.first().id, file=file_node, action='download').values_list('_id', 'total'))
  qs = FileVersion.includable_objects.filter(basefilenode__id=file_node.id).include('creator__guids').order_by('-created')
 
  for i, version in enumerate(qs):
@@ -233,7 +232,10 @@ def osfstorage_get_children(file_node, **kwargs):
  ) CHECKOUT_GUID ON TRUE
  LEFT JOIN LATERAL (
  SELECT P.total AS DOWNLOAD_COUNT FROM osf_pagecounter AS P
- WHERE P._id = 'download:' || %s || ':' || F._id
+ WHERE P.resource_id = %s
+ AND P.file_id = F.id
+ AND P.action = 'download'
+ AND P.version ISNULL
  LIMIT 1
  ) DOWNLOAD_COUNT ON TRUE
  LEFT JOIN LATERAL (
@@ -268,7 +270,7 @@ def osfstorage_get_children(file_node, **kwargs):
  AND (NOT F.type IN ('osf.trashedfilenode', 'osf.trashedfile', 'osf.trashedfolder'))
  """, [
  user_content_type_id,
- file_node.target._id,
+ file_node.target.guids.first().id,
  user_pk,
  user_pk,
  user_id,