diff --git a/Dockerfile b/Dockerfile index 8b39925..d9f016f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,57 +1,36 @@ -ARG image_url=gcr.io/panoptes-exp/panoptes-pocs -ARG image_tag=develop -FROM ${image_url}:${image_tag} AS pipeline-base - -LABEL description="Development environment for working with the PIPELINE" -LABEL maintainers="developers@projectpanoptes.org" -LABEL repo="github.com/panoptes/panoptes-pipeline" - -ARG userid=1000 -ENV USERID $userid - -ENV DEBIAN_FRONTEND=noninteractive -ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 -ENV PYTHONUNBUFFERED True - -ENV PORT 8080 - -USER "${userid}" - -USER "${USERID}" -WORKDIR /build -COPY --chown="${USERID}:${USERID}" . . -RUN echo "Building wheel" && \ - sudo chown -R "${userid}:${userid}" /build && \ - python setup.py bdist_wheel -d /build/dist - -FROM pipeline-base AS panoptes-pipeline - -USER "${USERID}" -WORKDIR /build -COPY --from=pipeline-base /build/dist/ /build/dist -RUN echo "Installing module" && \ - pip install --no-cache-dir "$(ls /build/dist/*.whl)" && \ +FROM continuumio/miniconda3 + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + astrometry.net source-extractor dcraw exiftool \ + libcfitsio-dev libcfitsio-bin \ + libpng-dev libjpeg-dev \ + libfreetype6-dev \ + libffi-dev && \ # Cleanup - pip cache purge && \ - conda clean -fay && \ - sudo apt-get autoremove --purge --yes \ - gcc pkg-config git && \ - sudo apt-get autoclean --yes && \ - sudo apt-get --yes clean && \ - sudo rm -rf /var/lib/apt/lists/* - -USER "${USERID}" -WORKDIR /app -COPY --chown="${USERID}:${USERID}" ./services/* /app/ -COPY ./notebooks/ProcessFITS.ipynb . -COPY ./notebooks/ProcessObservation.ipynb . - -RUN echo "Creating /input and /output directories" && \ - sudo mkdir -p /input && \ - sudo mkdir -p /output && \ - sudo chown -R "${USERID}:${USERID}" /input && \ - sudo chown -R "${USERID}:${USERID}" /output && \ - sudo chmod -R 777 /input && \ - sudo chmod -R 777 /output - -CMD [ "gunicorn --workers 1 --threads 8 --timeout 0 -k uvicorn.workers.UvicornWorker --bind :${PORT:-8080} pipeline:app" ] + apt-get autoremove --purge --yes && \ + apt-get autoclean --yes && \ + rm -rf /var/lib/apt/lists/* + +ADD http://data.astrometry.net/4100/index-4108.fits /usr/share/astrometry +ADD http://data.astrometry.net/4100/index-4110.fits /usr/share/astrometry +ADD http://data.astrometry.net/4100/index-4111.fits /usr/share/astrometry +ADD http://data.astrometry.net/4100/index-4112.fits /usr/share/astrometry +ADD http://data.astrometry.net/4100/index-4113.fits /usr/share/astrometry +ADD http://data.astrometry.net/4100/index-4114.fits /usr/share/astrometry +ADD http://data.astrometry.net/4100/index-4115.fits /usr/share/astrometry +ADD http://data.astrometry.net/4100/index-4116.fits /usr/share/astrometry +ADD http://data.astrometry.net/4100/index-4117.fits /usr/share/astrometry +ADD http://data.astrometry.net/4100/index-4118.fits /usr/share/astrometry +ADD http://data.astrometry.net/4100/index-4119.fits /usr/share/astrometry + +RUN conda update -n base conda && \ + conda install -c conda-forge mamba && \ + mamba install -c conda-forge \ + astropy astroplan astroquery photutils \ + scipy numpy pandas scikit-learn scikit-image numexpr \ + bokeh seaborn plotly panel \ + jupyterlab ipywidgets ipython-autotime \ + gcsfs google-cloud-storage \ + h5py \ + pip diff --git a/README.md b/README.md index 03de720..a21818a 100644 --- a/README.md +++ b/README.md @@ -4,22 +4,16 @@ PANOPTES Image Processing for the Extraction of Lightcurves in Nearby Exoplanets ## Description -So many planets, such little time... +This repository contains code that allows for the processing and discovering of [PANOPTES](https://www.projectpanoptes.org) data. -### Deploy +Most processing is done in the cloud using dedicated Google Cloud Platform services, but the code contained in this repository can be used to process results locally. -See [Deployment](../README.md#deploy) in main README for preferred deployment method. +In general the processing is set up as series of Jupyter Notebooks that are processed automatically using [papermill](https://papermill.readthedocs.io/en/latest/). The processed notebooks are stored as artifcats in storage buckets. -#### Notification Creation +## Usage -The bucket notification only needs to be created once, which can be done with the following command: +### Online -```sh -gsutil notification create -t process-raw-image -f json -e OBJECT_FINALIZE gs://panoptes-images-raw/ -``` +#### Binder -You can list existing notifications with: - -```sh -gsutil notification list gs://panoptes-images-incoming/ -``` \ No newline at end of file +[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/panoptes/panoptes-pipeline/prepare-cleanup) diff --git a/deploy.sh b/deploy.sh deleted file mode 100755 index 67f0c46..0000000 --- a/deploy.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -set -e - -TOPIC=${1:-panoptes-pipeline-image} -BASE_TAG=$(git rev-parse HEAD) -PROJECT_ID=panoptes-exp - -echo "Building image" -gcloud builds submit --tag "gcr.io/${PROJECT_ID}/${TOPIC}:${BASE_TAG}" . - -#gcloud compute instances update-container \ -# pipeline-processing \ -# --zone us-central1-a \ -# --container-image "gcr.io/${PROJECT_ID}/${TOPIC}:${BASE_TAG}" - -echo "Deploying ${TOPIC} to Cloud Run" -gcloud run deploy "${TOPIC}" \ - --region "us-west1" \ - --image "gcr.io/${PROJECT_ID}/${TOPIC}:${BASE_TAG}" \ - --no-allow-unauthenticated \ - --platform managed \ - --cpu 2 \ - --memory "8Gi" \ - --max-instances 500 \ - --concurrency 1 \ - --timeout "20m" - -echo "Deploying ${TOPIC/image/observation} to Cloud Run" -gcloud run deploy "${TOPIC/image/observation}" \ - --region "us-west1" \ - --image "gcr.io/${PROJECT_ID}/${TOPIC}:${BASE_TAG}" \ - --no-allow-unauthenticated \ - --platform managed \ - --cpu 2 \ - --memory "8Gi" \ - --max-instances 50 \ - --concurrency 1 \ - --timeout "20m" diff --git a/docker-compose.yaml b/docker-compose.yaml deleted file mode 100644 index 9df978c..0000000 --- a/docker-compose.yaml +++ /dev/null @@ -1,15 +0,0 @@ -services: - panoptes-pipeline: - image: "${IMAGE_NAME:-gcr.io/panoptes-exp/panoptes-pipeline}:${TAG_NAME:-latest}" - build: - context: . - dockerfile: ./Dockerfile - restart: never - init: true - tty: true - container_name: panoptes-pipeline - hostname: panoptes-pipeline - environment: - GOOGLE_APPLICATION_CREDENTIALS: - volumes: - - .:/output diff --git a/environment.yaml b/environment.yaml deleted file mode 100644 index 2423bc6..0000000 --- a/environment.yaml +++ /dev/null @@ -1,30 +0,0 @@ -channels: - - https://conda.anaconda.org/conda-forge -dependencies: - - astropy - - bottleneck - - fastapi - - gcsfs - - google-cloud-bigquery - - google-cloud-bigquery-storage - - google-cloud-pubsub - - google-cloud-storage - - h5py - - ipython-autotime - - matplotlib-base - - numexpr - - numpy - - pandas - - papermill - - photutils - - pip - - pyarrow - - tables - - tabulate - - scikit-learn - - scipy - - seaborn - - uvicorn - - pip: - - "git+https://github.com/panoptes/panoptes-utils@develop#egg=panoptes-utils[config,images,social]" - - google-cloud-firestore diff --git a/notebooks/ProcessFITS.ipynb b/notebooks/ProcessFITS.ipynb index d661c12..5a1c608 100644 --- a/notebooks/ProcessFITS.ipynb +++ b/notebooks/ProcessFITS.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -11,11 +11,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": { "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "time: 1.87 s\n" + ] + } + ], "source": [ "import tempfile\n", "from pathlib import Path\n", @@ -25,6 +33,9 @@ "import seaborn as sb\n", "from IPython.display import JSON, display, Markdown\n", "from astropy.wcs import WCS\n", + "from astropy.io import fits\n", + "from astropy.table import Table\n", + "from astropy.nddata import CCDData, Cutout2D\n", "from loguru import logger\n", "from panoptes.utils.images import bayer\n", "from panoptes.utils.images import fits as fits_utils\n", @@ -41,25 +52,51 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "