Merge branch 'release/19.29.0'

Author: brianjgeiger

CHANGELOG

@@ -2,6 +2,12 @@
 
 We follow the CalVer (https://calver.org/) versioning scheme: YY.MINOR.MICRO.
 
+19.29.0 (2019-10-02)
+===================
+- Use new pagecounter fields for increased query efficiency
+- Allow meetings to be sorted on download count
+- Remove old permissions fields now that we have Guardian
+
 19.28.0 (2019-09-24)
 ===================
 - API v2: Use consistent naming for JSON API type (kebab-case)

Dockerfile

@@ -1,99 +1,27 @@
-FROM python:2.7-slim-stretch
-
-ENV GOSU_VERSION=1.10 \
-    NODE_ENV=production \
-    NODE_VERSION=8.6.0 \
-    YARN_VERSION=1.1.0
-
-# Libraries such as matplotlib require a HOME directory for cache and configuration
-RUN set -ex \
-    && mkdir -p /var/www \
-    && chown www-data:www-data /var/www \
-    && apt-get update \
-    && apt-get install -y gnupg2 \
-    && for key in \
-      # GOSU
-      B42F6819007F00F88E364FD4036A9C25BF357DD4 \
-      # https://github.com/nodejs/docker-node/blob/9c25cbe93f9108fd1e506d14228afe4a3d04108f/8.2/Dockerfile
-      # gpg keys listed at https://github.com/nodejs/node#release-team
-      # Node
-      9554F04D7259F04124DE6B476D5A82AC7E37093B \
-      94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \
-      FD3A5288F042B6850C66B31F09FE44734EB7990E \
-      71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 \
-      DD8F2338BAE7501E3DD5AC78C273792F7D83545D \
-      B9AE9905FFD7803F25714661B63B535A4C206CA9 \
-      C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 \
-      56730D5401028683275BD23C23EFEFE93C4CFFFE \
-      # Yarn
-      6A010C5166006599AA17F08146C2130DFD2497F5 \
-    ; do \
-      gpg --keyserver hkp://ipv4.pool.sks-keyservers.net:80 --recv-keys "$key" || \
-      gpg --keyserver hkp://ha.pool.sks-keyservers.net:80 --recv-keys "$key" || \
-      gpg --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" || \
-      gpg --keyserver hkp://keyserver.pgp.com:80 --recv-keys "$key" \
-    ; done \
-    # Install dependancies
-    && apt-get install -y \
-        git \
-        libev4 \
-        libev-dev \
-        libevent-dev \
-        libxml2-dev \
-        libxslt1-dev \
-        zlib1g-dev \
-        curl \
-        # cryptography
-        build-essential \
-        libssl-dev \
-        libffi-dev \
-        python-dev \
-        # postgresql
-        libpq-dev \
-        # file audits
-        par2 \
-    && ARCH= \
-    && dpkgArch="$(dpkg --print-architecture)" \
-    && case "${dpkgArch##*-}" in \
-      amd64) ARCH='x64';; \
-      ppc64el) ARCH='ppc64le';; \
-      *) echo "unsupported architecture"; exit 1 ;; \
-    esac \
-    # grab gosu for easy step-down from root
-    && curl -o /usr/local/bin/gosu -SL "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch" \
-    && curl -o /usr/local/bin/gosu.asc -SL "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch.asc" \
-    && gpg --verify /usr/local/bin/gosu.asc \
-    && rm /usr/local/bin/gosu.asc \
-    && chmod +x /usr/local/bin/gosu \
-    # Node
-    && curl -SLO "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH.tar.xz" \
-    && curl -SLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \
-    && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc \
-    && grep " node-v$NODE_VERSION-linux-$ARCH.tar.xz\$" SHASUMS256.txt | sha256sum -c - \
-    && tar -xJf "node-v$NODE_VERSION-linux-$ARCH.tar.xz" -C /usr/local --strip-components=1 \
-    && rm "node-v$NODE_VERSION-linux-$ARCH.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt \
-    && ln -s /usr/local/bin/node /usr/local/bin/nodejs \
-    # Yarn
-    && curl -fSLO --compressed "https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz" \
-    && curl -fSLO --compressed "https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz.asc" \
-    && gpg --batch --verify yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz \
-    && mkdir -p /opt/yarn \
-    && tar -xzf yarn-v$YARN_VERSION.tar.gz -C /opt/yarn --strip-components=1 \
-    && ln -s /opt/yarn/bin/yarn /usr/local/bin/yarn \
-    && ln -s /opt/yarn/bin/yarn /usr/local/bin/yarnpkg \
-    && yarn global add bower \
-    && yarn cache clean \
-    && rm -rf \
-        yarn-v$YARN_VERSION.tar.gz.asc \
-        yarn-v$YARN_VERSION.tar.gz \
-        $HOME/.gnupg \
-        $HOME/.cache \
-    && apt-get remove -y \
-        curl \
-    && apt-get clean \
-    && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/* \
-    && mkdir -p /code
+FROM node:8-alpine
+
+# Source: https://github.com/docker-library/httpd/blob/7976cabe162268bd5ad2d233d61e340447bfc371/2.4/alpine/Dockerfile#L3
+RUN set -x \
+    && addgroup -g 82 -S www-data \
+    && adduser -h /var/www -u 82 -D -S -G www-data www-data
+
+RUN apk add --no-cache --virtual .run-deps \
+    su-exec \
+    bash \
+    python \
+    py-pip \
+    git \
+    # lxml2
+    libxml2 \
+    libxslt \
+    # psycopg2
+    postgresql-libs \
+    # cryptography
+    libffi \
+    # gevent
+    libev \
+    libevent \
+    && yarn global add bower
 
 WORKDIR /code
 
@@ -118,7 +46,22 @@ COPY ./addons/twofactor/requirements.txt ./addons/twofactor/
 #COPY ./addons/wiki/requirements.txt ./addons/wiki/
 COPY ./addons/zotero/requirements.txt ./addons/zotero/
 
-RUN for reqs_file in \
+RUN set -ex \
+    && mkdir -p /var/www \
+    && chown www-data:www-data /var/www \
+    && apk add --no-cache --virtual .build-deps \
+        build-base \
+        linux-headers \
+        python-dev \
+        # lxml2
+        musl-dev \
+        libxml2-dev \
+        libxslt-dev \
+        # psycopg2
+        postgresql-dev \
+        # cryptography
+        libffi-dev \
+    && for reqs_file in \
         /code/requirements.txt \
         /code/requirements/release.txt \
         /code/addons/*/requirements.txt \
@@ -128,17 +71,10 @@ RUN for reqs_file in \
     && (pip uninstall uritemplate.py --yes || true) \
     && pip install --no-cache-dir uritemplate.py==0.3.0 \
     # Fix: https://github.com/CenterForOpenScience/osf.io/pull/6783
-    && python -m compileall /usr/local/lib/python2.7 || true
-
-# OSF: Assets
-COPY ./.bowerrc ./bower.json ./
-RUN bower install --production --allow-root \
-    && bower cache clean --allow-root
-
-COPY ./package.json ./.yarnrc ./yarn.lock ./
-RUN yarn install --frozen-lockfile \
-    && yarn cache clean
+    && python -m compileall /usr/lib/python2.7 || true \
+    && apk del .build-deps
 
+# Settings
 COPY ./tasks/ ./tasks/
 COPY ./website/settings/ ./website/settings/
 COPY ./api/base/settings/ ./api/base/settings/
@@ -148,8 +84,29 @@ RUN mv ./website/settings/local-dist.py ./website/settings/local.py \
     && mv ./api/base/settings/local-dist.py ./api/base/settings/local.py \
     && sed 's/DEBUG_MODE = True/DEBUG_MODE = False/' -i ./website/settings/local.py
 
+# Bower Assets
+COPY ./.bowerrc ./bower.json ./
+COPY ./admin/.bowerrc ./admin/bower.json ./admin/
+RUN \
+    # OSF
+    bower install --production --allow-root \
+    && bower cache clean --allow-root \
+    # Admin
+    && cd ./admin \
+    && bower install --production --allow-root \
+    && bower cache clean --allow-root
+
+# Webpack Assets
+#
+## OSF
+COPY ./package.json ./.yarnrc ./yarn.lock ./
 COPY ./webpack* ./
 COPY ./website/static/ ./website/static/
+## Admin
+COPY ./admin/package.json ./admin/yarn.lock ./admin/
+COPY ./admin/webpack* ./admin/
+COPY ./admin/static/ ./admin/static/
+## Addons
 COPY ./addons/bitbucket/static/ ./addons/bitbucket/static/
 COPY ./addons/box/static/ ./addons/box/static/
 COPY ./addons/citations/static/ ./addons/citations/static/
@@ -168,34 +125,29 @@ COPY ./addons/s3/static/ ./addons/s3/static/
 COPY ./addons/twofactor/static/ ./addons/twofactor/static/
 COPY ./addons/wiki/static/ ./addons/wiki/static/
 COPY ./addons/zotero/static/ ./addons/zotero/static/
-RUN mkdir -p ./website/static/built/ \
+RUN \
+    # OSF
+    yarn install --frozen-lockfile \
+    && mkdir -p ./website/static/built/ \
     && invoke build_js_config_files \
-    && yarn run webpack-prod
-# /OSF: Assets
-
-# Admin: Assets
-COPY ./admin/.bowerrc ./admin/bower.json ./admin/
-RUN cd ./admin \
-    && bower install --production --allow-root \
-    && bower cache clean --allow-root
-
-COPY ./admin/package.json ./admin/yarn.lock ./admin/
-RUN cd ./admin \
+    && yarn run webpack-prod \
+    # Admin
+    && cd ./admin \
     && yarn install --frozen-lockfile \
-    && yarn cache clean
-
-COPY ./admin/webpack* ./admin/
-COPY ./admin/static/ ./admin/static/
-RUN cd ./admin \
-    && yarn run webpack-prod
-# /Admin: Assets
+    && yarn run webpack-prod \
+    && cd ../ \
+    # Cleanup
+    && yarn cache clean \
+    && npm cache clean --force
 
 # Copy the rest of the code over
 COPY ./ ./
 
 ARG GIT_COMMIT=
 ENV GIT_COMMIT ${GIT_COMMIT}
 
+# TODO: Admin/API should fully specify their bower static deps, and not include ./website/static in their defaults.py.
+#       (this adds an additional 300+mb to the build image)
 RUN for module in \
         api.base.settings \
         admin.base.settings \
@@ -211,4 +163,4 @@ RUN for module in \
     ; done \
     && rm ./website/settings/local.py ./api/base/settings/local.py
 
-CMD ["gosu", "nobody", "invoke", "--list"]
+CMD ["su-exec", "nobody", "invoke", "--list"]

addons/base/views.py

@@ -330,9 +330,9 @@ def get_auth(auth, **kwargs):
                     # version index is 0 based
                     version_index = version - 1
                     if action == 'render':
-                        update_analytics(node, file_id, version_index, 'view')
+                        update_analytics(node, filenode, version_index, 'view')
                     elif action == 'download' and not from_mfr:
-                        update_analytics(node, file_id, version_index, 'download')
+                        update_analytics(node, filenode, version_index, 'download')
                     if waffle.switch_is_active(features.ELASTICSEARCH_METRICS):
                         if isinstance(node, Preprint):
                             metric_class = get_metric_class_for_action(action, from_mfr=from_mfr)
@@ -517,7 +517,6 @@ def create_waterbutler_log(payload, **kwargs):
 def addon_delete_file_node(self, target, user, event_type, payload):
     """ Get addon BaseFileNode(s), move it into the TrashedFileNode collection
     and remove it from StoredFileNode.
-
     Required so that the guids of deleted addon files are not re-pointed when an
     addon file or folder is moved or renamed.
     """

addons/osfstorage/tests/test_models.py

@@ -184,9 +184,9 @@ def test_download_count_file(self, mock_session):
         mock_session.data = {}
         child = self.node_settings.get_root().append_file('Test')
 
-        utils.update_analytics(self.project, child._id, 0)
-        utils.update_analytics(self.project, child._id, 1)
-        utils.update_analytics(self.project, child._id, 2)
+        utils.update_analytics(self.project, child, 0)
+        utils.update_analytics(self.project, child, 1)
+        utils.update_analytics(self.project, child, 2)
 
         assert_equals(child.get_download_count(), 3)
         assert_equals(child.get_download_count(0), 1)

addons/osfstorage/tests/test_utils.py

@@ -31,9 +31,9 @@ def setUp(self):
 
     def test_serialize_revision(self):
         sessions.sessions[request._get_current_object()] = Session()
-        utils.update_analytics(self.project, self.record._id, 0)
-        utils.update_analytics(self.project, self.record._id, 0)
-        utils.update_analytics(self.project, self.record._id, 2)
+        utils.update_analytics(self.project, self.record, 0)
+        utils.update_analytics(self.project, self.record, 0)
+        utils.update_analytics(self.project, self.record, 2)
         expected = {
             'index': 1,
             'user': {
@@ -58,9 +58,9 @@ def test_serialize_revision(self):
 
     def test_anon_revisions(self):
         sessions.sessions[request._get_current_object()] = Session()
-        utils.update_analytics(self.project, self.record._id, 0)
-        utils.update_analytics(self.project, self.record._id, 0)
-        utils.update_analytics(self.project, self.record._id, 2)
+        utils.update_analytics(self.project, self.record, 0)
+        utils.update_analytics(self.project, self.record, 0)
+        utils.update_analytics(self.project, self.record, 2)
         expected = {
             'index': 2,
             'user': None,

addons/osfstorage/utils.py

@@ -16,24 +16,28 @@
 LOCATION_KEYS = ['service', settings.WATERBUTLER_RESOURCE, 'object']
 
 
-def update_analytics(node, file_id, version_idx, action='download'):
+def update_analytics(node, file, version_idx, action='download'):
     """
     :param Node node: Root node to update
     :param str file_id: The _id field of a filenode
     :param int version_idx: Zero-based version index
     :param str action: is this logged as download or a view
     """
-    # Pass in contributors to check that contributors' downloads
+    # Pass in contributors and group members to check that their downloads
     # do not count towards total download count
     contributors = []
-    if node.contributors:
+    if getattr(node, 'contributors_and_group_members', None):
+        contributors = node.contributors_and_group_members
+    elif getattr(node, 'contributors', None):
         contributors = node.contributors
+
     node_info = {
         'contributors': contributors
     }
+    resource = node.guids.first()
 
-    update_counter('{0}:{1}:{2}'.format(action, node._id, file_id), node_info=node_info)
-    update_counter('{0}:{1}:{2}:{3}'.format(action, node._id, file_id, version_idx), node_info=node_info)
+    update_counter(resource, file, version=None, action=action, node_info=node_info)
+    update_counter(resource, file, version_idx, action, node_info=node_info)
 
 
 def serialize_revision(node, record, version, index, anon=False):
@@ -44,7 +48,6 @@ def serialize_revision(node, record, version, index, anon=False):
     :param FileVersion version: The version to serialize
     :param int index: One-based index of version
     """
-
     if anon:
         user = None
     else:

addons/osfstorage/views.py

@@ -106,8 +106,7 @@ def osfstorage_get_revisions(file_node, payload, target, **kwargs):
     counter_prefix = 'download:{}:{}:'.format(file_node.target._id, file_node._id)
 
     version_count = file_node.versions.count()
-    # Don't worry. The only % at the end of the LIKE clause, the index is still used
-    counts = dict(PageCounter.objects.filter(_id__startswith=counter_prefix).values_list('_id', 'total'))
+    counts = dict(PageCounter.objects.filter(resource=file_node.target.guids.first().id, file=file_node, action='download').values_list('_id', 'total'))
     qs = FileVersion.includable_objects.filter(basefilenode__id=file_node.id).include('creator__guids').order_by('-created')
 
     for i, version in enumerate(qs):
@@ -233,7 +232,10 @@ def osfstorage_get_children(file_node, **kwargs):
             ) CHECKOUT_GUID ON TRUE
             LEFT JOIN LATERAL (
                 SELECT P.total AS DOWNLOAD_COUNT FROM osf_pagecounter AS P
-                WHERE P._id = 'download:' || %s || ':' || F._id
+                WHERE P.resource_id = %s
+                AND P.file_id = F.id
+                AND P.action = 'download'
+                AND P.version ISNULL
                 LIMIT 1
             ) DOWNLOAD_COUNT ON TRUE
             LEFT JOIN LATERAL (
@@ -268,7 +270,7 @@ def osfstorage_get_children(file_node, **kwargs):
             AND (NOT F.type IN ('osf.trashedfilenode', 'osf.trashedfile', 'osf.trashedfolder'))
         """, [
             user_content_type_id,
-            file_node.target._id,
+            file_node.target.guids.first().id,
             user_pk,
             user_pk,
             user_id,