first potentially working version of sam downloads

mpcontribs-portal/Dockerfile

@@ -56,7 +56,7 @@ COPY --from=python-builds /app/mpcontribs /app/mpcontribs
 COPY --from=python-builds /app/supervisord /app/supervisord
 COPY --from=webpack /app/webpack-stats.json /app/webpack-stats.json
 
-RUN apt-get update && apt-get install -y --no-install-recommends supervisor && apt-get clean
+RUN apt-get update && apt-get install -y --no-install-recommends supervisor jq && apt-get clean
 WORKDIR /app
 RUN mkdir -p /var/log/supervisor
 ENV PATH=/root/.local/bin:$PATH

mpcontribs-portal/mpcontribs/portal/assets/js/landingpage.js

@@ -381,32 +381,38 @@ function prep_download(query, prefix) {
     $.get({
         contentType: "json", dataType: "json", url: url, data: query
     }).done(function(response) {
+        const fmt = query["format"];
+        $("#" + prefix + "download_" + fmt).removeClass('is-loading').addClass("is-hidden");
         if ("error" in response) {
             alert(response["error"]);
-        } else if (response["status"] == "READY") {
+        } else if (response["status"] === "ERROR") {
+            alert("Error during download generation for" + response["redis_key"]);
+        } else if (response["status"] === "READY") {
             const href = "/contributions/download/get?" + $.param(query);
             $("#" + prefix + "get_download").attr("href", href).removeClass("is-hidden");
-            const fmt = query["format"];
-            $("#" + prefix + "download_" + fmt).removeClass('is-loading').addClass("is-hidden");
             $("#" + prefix + "check_download").addClass("is-hidden");
             $("#" + prefix + "download_progress").addClass("is-hidden");
-        } else { // SUBMITTED, UNDEFINED or ONGOING
-            $("#" + prefix + "check_download").click(function() { prep_download(query, prefix); });
+        } else if (response["status"] === "SUBMITTED" || response["status"] === "UNDEFINED") {
             $("#" + prefix + "check_download").removeClass("is-hidden");
-            $("#" + prefix + "download_progress").text(response["status"]).removeClass("is-hidden");
+            $("#" + prefix + "download_progress").removeClass("is-hidden");
+        } else { // percent time elapsed
+            $("#" + prefix + "check_download").removeClass("is-hidden");
+            $("#" + prefix + "download_progress").attr("value", response["status"]).removeClass("is-hidden");
         }
     });
 }
 
 $('a[name="download"]').click(function(e) {
     $('a[name="download"]').addClass("is-hidden");
+    $('a[name="include"]').addClass("is-hidden");
     $(this).addClass('is-loading').removeClass("is-hidden");
     const fmt = $(this).data('format');
     const include = $('input[name="include"]:checked').map(function() {
         return $(this).val();
     }).get().join(',');
     var download_query = {"format": fmt, "project": project};
     if (include) { download_query["include"] = include; }
+    $("#check_download").click(function() { prep_download(download_query, ""); });
     prep_download(download_query, "");
 });
 

mpcontribs-portal/mpcontribs/portal/templates/landingpage.html

@@ -69,10 +69,16 @@
                             <label for="with_{{component}}"><span>with {{component}}</span></label>
                         </div>
                         {% endfor %}
-                        <div class="dropdown-item" style="white-space: nowrap;">
-                            <a class="button is-primary is-light is-small" name=download id=download_json data-format=json>JSON</a>
-                            <a class="button is-primary is-light is-small" name=download id=download_csv data-format=csv>CSV</a>
-                            <span class="tag is-hidden" id=download_progress></span>
+                        <div class="dropdown-item">
+                            <div class="field is-grouped">
+                                <p class="control">
+                                <a class="button is-primary is-light is-small" name=download id=download_json data-format=json>JSON</a>
+                                </p>
+                                <p class="control">
+                                <a class="button is-primary is-light is-small" name=download id=download_csv data-format=csv>CSV</a>
+                                </p>
+                            </div>
+                            <progress class="progress is-small is-hidden" max="100" id=download_progress></progress>
                             <a class="button is-warning is-light is-small is-hidden" id=check_download>Check status</a>
                             <a class="button is-danger is-light is-small is-hidden" id=get_download>Click to download</a>
                         </div>

mpcontribs-portal/mpcontribs/portal/views.py

@@ -35,7 +35,7 @@
 j2h = Json2Html()
 s3_client = boto3.client('s3')
 lambda_client = boto3.client('lambda')
-redis_store = Redis.from_url("redis://" + os.environ["REDIS_ADDRESS"])
+redis_store = Redis.from_url("redis://" + os.environ["REDIS_ADDRESS"], decode_responses=True)
 
 
 def visit(path, key, value):
@@ -492,24 +492,24 @@ def make_download(headers, query, include=None):
         except ClientError:
             next_version = 1  # about to generate first version
 
-        filename = _get_filename(query, include),
+        filename = _get_filename(query, include)
         fmt = query.get("format", "json")
         redis_key = f"{BUCKET}:{filename}:{fmt}:{next_version}"
+        json_resp["redis_key"] = redis_key
         status = redis_store.get(redis_key)
 
         if status is None:
+            payload={
+                "redis_key": redis_key,
+                "host": os.environ["MPCONTRIBS_CLIENT_HOST"],
+                "headers": headers,
+                "query": query,
+                "include": include
+            }
             response = lambda_client.invoke(
                 FunctionName='MPContribsMakeDownloadFunction',
                 InvocationType='Event',
-                Payload={
-                    "redis_key": redis_key,
-                    "redis": os.environ["REDIS_ADDRESS"],
-                    # TODO PRODUCTION get this host's private IP address and determine correct api port
-                    "host": os.environ["MPCONTRIBS_API_HOST"],
-                    "headers": headers,
-                    "query": query,
-                    "include": include
-                }
+                Payload=payload
             )
             if response["StatusCode"] == 202:
                 status = "SUBMITTED"

mpcontribs-portal/requirements.txt

@@ -8,12 +8,14 @@ django-webpack4-loader==0.0.5
 fastnumbers==3.2.1
 gunicorn[gevent]==20.1.0
 ipykernel==6.9.2
+ipython-genutils==0.2.0
 jinja2==3.0.3
 json2html==1.3.0
 monty==2022.3.12
 mpcontribs-client==4.2.9
 nbconvert==5.6.1
 nbformat==5.2.0
+redis==4.1.4
 scipy==1.8.0
 setproctitle==1.2.2
 whitenoise==6.0.0

mpcontribs-portal/start.sh

@@ -4,6 +4,13 @@ zzz=$(($DEPLOYMENT*60))
 echo "$SUPERVISOR_PROCESS_NAME: waiting for $zzz seconds before start..."
 sleep $zzz
 
+if [ ! -z "$METADATA_URI" ]; then
+    task_ip=`curl ${METADATA_URI}/task | jq -r '.Networks[0].IPv4Addresses[0]'`
+    export MPCONTRIBS_CLIENT_HOST=$task_ip:$MPCONTRIBS_API_PORT
+else
+    export MPCONTRIBS_CLIENT_HOST=$MPCONTRIBS_API_HOST
+fi
+
 pmgrc=$HOME/.pmgrc.yaml
 [[ ! -e $pmgrc ]] && echo "PMG_DUMMY_VAR: dummy" > $pmgrc
 python manage.py migrate --noinput

mpcontribs-portal/supervisord/supervisord.conf.jinja

@@ -8,6 +8,9 @@ environment=
     AWS_ACCESS_KEY_ID="%(ENV_AWS_ACCESS_KEY_ID)s",
     AWS_SECRET_ACCESS_KEY="%(ENV_AWS_SECRET_ACCESS_KEY)s",
     {% endif %}
+    METADATA_URI="%(ENV_ECS_CONTAINER_METADATA_URI_V4)s",
+    AWS_REGION="us-east-1",
+    AWS_DEFAULT_REGION="us-east-1",
     DD_PROFILING_ENABLED="true",
     DD_LOGS_INJECTION="true",
     PYTHONUNBUFFERED=1,
@@ -56,6 +59,7 @@ environment=
     DEPLOYMENT={{ loop.index0 }},
     PORTAL_PORT={{ cfg.portal_port }},
     MPCONTRIBS_API_HOST="{{ mpcontribs_api_host }}:{{ cfg.api_port }}",
+    MPCONTRIBS_API_PORT="{{ cfg.api_port }}",
     TRADEMARK="{{ cfg.tm }}",
     S3_DOWNLOADS_BUCKET="mpcontribs-downloads-{{ cfg.s3 }}",
     ADMIN_GROUP="admin_{{ deployment }}.materialsproject.org"

mpcontribs-sam-download/events/event.json

@@ -1,6 +1,5 @@
 {
     "redis_key": "mpcontribs-downloads:lambda-sandbox-test:json:0",
-    "redis": "redis/13",
     "host": "contribs-apis:10000",
     "headers": {
         "X-Consumer-Username": "google:[email protected]",

mpcontribs-sam-download/make_download/app.py

@@ -10,52 +10,53 @@
 from mpcontribs.client import Client
 
 logger = logging.getLogger()
-logger.setLevel(logging.DEBUG)
+logger.setLevel(os.environ["MPCONTRIBS_CLIENT_LOG_LEVEL"])
 s3_client = boto3.client('s3')
+timeout = int(os.environ["LAMBDA_TIMEOUT"])
+redis_address = os.environ["REDIS_ADDRESS"]
+store = Redis.from_url(f"redis://{redis_address}")
+store.ping()
 
+def get_remaining(event, context):
+    remaining = context.get_remaining_time_in_millis() / 1000. - 0.5
+    if remaining < 3:
+        raise ValueError("TIMEOUT in 3s!")
+
+    elapsed_pct = (timeout - remaining) / timeout * 100.
+    store.set(event["redis_key"], f"{elapsed_pct:.1f}")
+    return remaining
 
-def lambda_handler(event, context):
-    logger.info(f"Received event: {event}")
-    redis_key = event["redis_key"]  # "{bucket}:{filename}:{fmt}:{version}"
-    store = Redis.from_url("redis://" + event["redis"])
-    store.ping()
-    store.set(redis_key, "ONGOING")
 
+def lambda_handler(event, context):
+    get_remaining(event, context)
     query, include = event["query"], event["include"]
     project = query["project"]
-    bucket, filename, fmt, version = redis_key.split(":")
+    bucket, filename, fmt, version = event["redis_key"].split(":")
 
     try:
         client = Client(
             host=event["host"], headers=event["headers"], project=project
         )
-        logger.info(client.get_totals())
-        remaining = context.get_remaining_time_in_millis() / 1000. - 0.5
-        logger.debug(f"REMAINING {remaining:.1f}s")
-        all_ids = client.get_all_ids(
-            query=query, include=include, timeout=remaining/2.
-        ).get(project)
-        logger.info(f"ALL_IDS {len(all_ids)}")
+        remaining = get_remaining(event, context)
         tmpdir = Path("/tmp")
         outdir = tmpdir / filename
-        remaining = context.get_remaining_time_in_millis() / 1000. - 0.5
-        logger.debug(f"REMAINING {remaining:.1f}s")
         ndownloads = client.download_contributions(
             query=query, include=include, outdir=outdir, timeout=remaining
         )
-        logger.info(f"DOWNLOADS: {ndownloads}")
+        get_remaining(event, context)
         make_archive(outdir, "zip", outdir)
+        get_remaining(event, context)
         zipfile = outdir.with_suffix(".zip")
         resp = zipfile.read_bytes()
         s3_client.put_object(
             Bucket=bucket, Key=f"{filename}_{fmt}.zip",
             Metadata={"version": version},
             Body=resp, ContentType="application/zip"
         )
+        get_remaining(event, context)
         rmtree(outdir)
         os.remove(zipfile)
-        logger.info("DONE")
-        store.set(redis_key, "READY")
+        store.set(event["redis_key"], "READY")
     except Exception as e:
-        logger.error(f"Exception: {e}", exc_info=True)
-        store.set(redis_key, "ERROR")
+        logger.error(str(e), exc_info=True)
+        store.set(event["redis_key"], "ERROR")

mpcontribs-sam-download/template.yaml

@@ -1,12 +1,30 @@
 AWSTemplateFormatVersion: '2010-09-09'
 Transform: AWS::Serverless-2016-10-31
 
+Parameters:
+  LogLevel:
+    Type: String
+    AllowedValues:
+      - DEBUG
+      - INFO
+      - WARNING
+    Default: WARNING
+  RedisAddress:
+    Type: String
+    Default: redis/13
+  MemorySize:
+    Type: Number
+    Default: 1024
+  Timeout:
+    Type: Number
+    Default: 900
+
 Globals:
   Function:
     Handler: app.lambda_handler
     Runtime: python3.9
-    MemorySize: 1024
-    Timeout: 900
+    MemorySize: !Ref MemorySize
+    Timeout: !Ref Timeout
     EventInvokeConfig:
       MaximumRetryAttempts: 0
     VpcConfig:
@@ -25,7 +43,9 @@ Resources:
       CodeUri: make_download/
       Environment:
         Variables:
-          MPCONTRIBS_CLIENT_LOG_LEVEL: DEBUG
+          MPCONTRIBS_CLIENT_LOG_LEVEL: !Ref LogLevel
+          REDIS_ADDRESS: !Ref RedisAddress
+          LAMBDA_TIMEOUT: !Ref Timeout
 
 Outputs:
   MPContribsMakeDownloadFunction: