Skip to content

Commit

Permalink
Fix CI
Browse files Browse the repository at this point in the history
  • Loading branch information
sfodagain committed Nov 22, 2023
1 parent 26629a0 commit e1f8308
Show file tree
Hide file tree
Showing 4 changed files with 145 additions and 95 deletions.
50 changes: 22 additions & 28 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -90,10 +90,10 @@ jobs:
aws-region: ${{ env.AWS_DEFAULT_REGION }}
- name: run PubSub sample
run: |
python ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_pubsub_cfg.json
python ./aws-iot-device-sdk-js-v2/utils/run_in_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_pubsub_cfg.json
- name: run Windows Certificate Connect sample
run: |
python ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_windows_cert_connect_cfg.json
python ./aws-iot-device-sdk-js-v2/utils/run_in_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_windows_cert_connect_cfg.json
- name: configure AWS credentials (Device Advisor)
uses: aws-actions/configure-aws-credentials@v1
with:
Expand Down Expand Up @@ -124,14 +124,14 @@ jobs:
aws-region: ${{ env.AWS_DEFAULT_REGION }}
- name: run PubSub sample
run: |
python3 ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_pubsub_cfg.json
python3 ./aws-iot-device-sdk-js-v2/utils/run_in_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_pubsub_cfg.json
- name: run PKCS12 sample
run: |
cert=$(aws secretsmanager get-secret-value --region us-east-1 --secret-id "ci/PubSub/cert" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo -e "$cert" > /tmp/certificate.pem
key=$(aws secretsmanager get-secret-value --region us-east-1 --secret-id "ci/PubSub/key" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo -e "$key" > /tmp/privatekey.pem
pkcs12_password=$(aws secretsmanager get-secret-value --region us-east-1 --secret-id "ci/PubSub/key_pkcs12_password" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\")
openssl pkcs12 -export -in /tmp/certificate.pem -inkey /tmp/privatekey.pem -out /tmp/pkcs12-key.p12 -name PubSub_Thing_Alias -password pass:$pkcs12_password
python3 ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_pkcs12_connect_cfg.json
python3 ./aws-iot-device-sdk-js-v2/utils/run_in_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_pkcs12_connect_cfg.json
- name: configure AWS credentials (Device Advisor)
uses: aws-actions/configure-aws-credentials@v1
with:
Expand Down Expand Up @@ -162,7 +162,7 @@ jobs:
aws-region: ${{ env.AWS_DEFAULT_REGION }}
- name: run PubSub sample
run: |
python3 ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_pubsub_cfg.json
python3 ./aws-iot-device-sdk-js-v2/utils/run_in_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_pubsub_cfg.json
- name: configure AWS credentials (Device Advisor)
uses: aws-actions/configure-aws-credentials@v1
with:
Expand Down Expand Up @@ -197,104 +197,98 @@ jobs:
aws-region: ${{ env.AWS_DEFAULT_REGION }}
- name: run Basic Connect sample
run: |
python3 ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_basic_connect_cfg.json
python3 ./aws-iot-device-sdk-js-v2/utils/run_in_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_basic_connect_cfg.json
- name: run Websocket Connect sample
run: |
python3 ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_websocket_connect_cfg.json
python3 ./aws-iot-device-sdk-js-v2/utils/run_in_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_websocket_connect_cfg.json
- name: run PubSub sample
run: |
python3 ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_pubsub_cfg.json
python3 ./aws-iot-device-sdk-js-v2/utils/run_in_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_pubsub_cfg.json
- name: run PubSub JS sample
run: |
python3 ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_pubsub_js_cfg.json
python3 ./aws-iot-device-sdk-js-v2/utils/run_in_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_pubsub_js_cfg.json
- name: run PKCS11 Connect sample
run: |
mkdir -p /tmp/tokens
export SOFTHSM2_CONF=/tmp/softhsm2.conf
echo "directories.tokendir = /tmp/tokens" > /tmp/softhsm2.conf
python3 ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_pkcs11_connect_cfg.json
python3 ./aws-iot-device-sdk-js-v2/utils/run_in_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_pkcs11_connect_cfg.json
- name: configure AWS credentials (Cognito)
uses: aws-actions/configure-aws-credentials@v1
with:
role-to-assume: ${{ env.CI_COGNITO_ROLE }}
aws-region: ${{ env.AWS_DEFAULT_REGION }}
- name: run Cognito Connect sample
run: |
python3 ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_cognito_connect_cfg.json
python3 ./aws-iot-device-sdk-js-v2/utils/run_in_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_cognito_connect_cfg.json
- name: configure AWS credentials (X509)
uses: aws-actions/configure-aws-credentials@v1
with:
role-to-assume: ${{ env.CI_X509_ROLE }}
aws-region: ${{ env.AWS_DEFAULT_REGION }}
- name: run X509 sample
run: |
python3 ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_x509_connect_cfg.json
python3 ./aws-iot-device-sdk-js-v2/utils/run_in_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_x509_connect_cfg.json
- name: configure AWS credentials (Custom Authorizer)
uses: aws-actions/configure-aws-credentials@v1
with:
role-to-assume: ${{ env.CI_CUSTOM_AUTHORIZER_ROLE }}
aws-region: ${{ env.AWS_DEFAULT_REGION }}
- name: run CustomAuthorizerConnect sample
run: |
python3 ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_custom_authorizer_connect_cfg.json
python3 ./aws-iot-device-sdk-js-v2/utils/run_in_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_custom_authorizer_connect_cfg.json
- name: configure AWS credentials (Shadow)
uses: aws-actions/configure-aws-credentials@v1
with:
role-to-assume: ${{ env.CI_SHADOW_ROLE }}
aws-region: ${{ env.AWS_DEFAULT_REGION }}
- name: run Shadow sample
run: |
python3 ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_shadow_cfg.json
python3 ./aws-iot-device-sdk-js-v2/utils/run_in_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_shadow_cfg.json
- name: run Mqtt5 Shadow sample
run: |
python3 ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_shadow_mqtt5_cfg.json
python3 ./aws-iot-device-sdk-js-v2/utils/run_in_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_shadow_mqtt5_cfg.json
- name: configure AWS credentials (Jobs)
uses: aws-actions/configure-aws-credentials@v1
with:
role-to-assume: ${{ env.CI_JOBS_ROLE }}
aws-region: ${{ env.AWS_DEFAULT_REGION }}
- name: run Jobs sample
run: |
python3 ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_jobs_cfg.json
python3 ./aws-iot-device-sdk-js-v2/utils/run_in_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_jobs_cfg.json
- name: run Mqtt5 Jobs sample
run: |
python3 ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_jobs_mqtt5_cfg.json
python3 ./aws-iot-device-sdk-js-v2/utils/run_in_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_jobs_mqtt5_cfg.json
- name: configure AWS credentials (Fleet provisioning)
uses: aws-actions/configure-aws-credentials@v1
with:
role-to-assume: ${{ env.CI_FLEET_PROVISIONING_ROLE }}
aws-region: ${{ env.AWS_DEFAULT_REGION }}
- name: run Fleet Provisioning sample
run: |
echo "Generating UUID for IoT thing"
Sample_UUID=$(python3 -c "import uuid; print (uuid.uuid4())")
python3 ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_fleet_provisioning_cfg.json --input_uuid ${Sample_UUID}
python3 ./aws-iot-device-sdk-js-v2/utils/delete_iot_thing_ci.py --thing_name "Fleet_Thing_${Sample_UUID}" --region "us-east-1"
python3 ./aws-iot-device-sdk-js-v2/utils/run_fleet_provisioning_sample.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_fleet_provisioning_cfg.json --thing-name-prefix "Fleet_Thing_"
- name: run Mqtt5 Fleet Provisioning sample
run: |
echo "Generating UUID for IoT thing"
Sample_UUID=$(python3 -c "import uuid; print (uuid.uuid4())")
python3 ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_fleet_provisioning_mqtt5_cfg.json --input_uuid ${Sample_UUID}
python3 ./aws-iot-device-sdk-js-v2/utils/delete_iot_thing_ci.py --thing_name "Fleet_Thing_${Sample_UUID}" --region "us-east-1"
python3 ./aws-iot-device-sdk-js-v2/utils/run_fleet_provisioning_sample.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_fleet_provisioning_mqtt5_cfg.json --thing-name-prefix "Fleet_Thing_"
- name: configure AWS credentials (MQTT5 PubSub)
uses: aws-actions/configure-aws-credentials@v1
with:
role-to-assume: ${{ env.CI_MQTT5_ROLE }}
aws-region: ${{ env.AWS_DEFAULT_REGION }}
- name: run MQTT5 PubSub sample
run: |
python3 ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_mqtt5_pubsub_cfg.json
python3 ./aws-iot-device-sdk-js-v2/utils/run_in_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_mqtt5_pubsub_cfg.json
- name: run MQTT5 Shared Subscription sample
run: |
python3 ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_mqtt5_shared_subscription_cfg.json
python3 ./aws-iot-device-sdk-js-v2/utils/run_in_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_mqtt5_shared_subscription_cfg.json
- name: configure AWS credentials (Greengrass)
uses: aws-actions/configure-aws-credentials@v1
with:
role-to-assume: ${{ env.CI_GREENGRASS_ROLE }}
aws-region: ${{ env.AWS_DEFAULT_REGION }}
- name: run Greengrass Discovery sample
run: |
python3 ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_greengrass_discovery_cfg.json
python3 ./aws-iot-device-sdk-js-v2/utils/run_in_ci.py --file ./aws-iot-device-sdk-js-v2/.github/workflows/ci_run_greengrass_discovery_cfg.json
# check that docs can still build
check-docs:
Expand Down
82 changes: 82 additions & 0 deletions utils/ci_iot_thing.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0.

import sys

import boto3


def create_iot_thing(thing_name, region, policy_name, certificate_path, key_path, thing_group=None):
""" Create IoT thing along with policy and credentials. """

iot_client = boto3.client('iot', region_name=region)

print(f"Creating thing '{thing_name}'", file=sys.stderr)

iot_client.create_thing(thingName=thing_name)
if thing_group:
iot_client.add_thing_to_thing_group(thingGroupName=thing_group, thingName=thing_name)

try:
print("Creating certificate", file=sys.stderr)
create_cert_response = iot_client.create_keys_and_certificate(
setAsActive=True
)

f = open(certificate_path, "w")
f.write(create_cert_response['certificatePem'])
f.close()

f = open(key_path, "w")
f.write(create_cert_response['keyPair']['PrivateKey'])
f.close()

certificate_arn = create_cert_response['certificateArn']

print("Attaching policy to certificate", file=sys.stderr)
iot_client.attach_policy(policyName=policy_name, target=certificate_arn)

print("Attaching certificate to thing", file=sys.stderr)
iot_client.attach_thing_principal(thingName=thing_name, principal=certificate_arn)
except Exception:
try:
iot_client.delete_thing(thingName=thing_name)
except Exception:
print("ERROR: Could not delete thing", file=sys.stderr)
raise

print("IoT thing created successfully", file=sys.stderr)


def delete_iot_thing(thing_name, region):
""" Delete IoT thing and all its principals. """

try:
iot_client = boto3.client('iot', region_name=region)
except Exception as e:
print(f"ERROR: Could not make Boto3 client. Credentials likely could not be sourced", file=sys.stderr)
raise

# Detach and delete thing's principals.
try:
thing_principals = iot_client.list_thing_principals(thingName=thing_name)
print(f"Detaching and deleting principals: {thing_principals}", file=sys.stderr)
for principal in thing_principals["principals"]:
certificate_id = principal.split("/")[1]
iot_client.detach_thing_principal(thingName=thing_name, principal=principal)
iot_client.update_certificate(certificateId=certificate_id, newStatus='INACTIVE')
iot_client.delete_certificate(certificateId=certificate_id, forceDelete=True)
except Exception:
print("ERROR: Could not delete certificate for IoT thing {thing_name}, probably thing does not exist",
file=sys.stderr)
raise

# Delete thing.
try:
iot_client.delete_thing(thingName=thing_name)
except Exception:
raise

print("IoT thing deleted successfully", file=sys.stderr)

return 0
67 changes: 0 additions & 67 deletions utils/delete_iot_thing_ci.py

This file was deleted.

41 changes: 41 additions & 0 deletions utils/run_fleet_provisioning_sample.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0.

import argparse
import uuid
import os
import sys
import run_in_ci
import ci_iot_thing


def main():
argument_parser = argparse.ArgumentParser(
description="Run Fleet Provisioning sample in CI")
argument_parser.add_argument("--file", required=True, help="Configuration file to pull CI data from")
argument_parser.add_argument(
"--input-uuid", required=False,
help="Use specified UUID for thing name instead of a random one")
argument_parser.add_argument(
"--thing-name-prefix", required=False, default="",
help="Prefix for a thing name, should be the same that Fleet Provisioning template uses")
argument_parser.add_argument(
"--region", required=False, default="us-east-1", help="The name of the region to use")
parsed_commands = argument_parser.parse_args()

cfg_file = parsed_commands.file
thing_uuid = parsed_commands.input_uuid if parsed_commands.input_uuid else str(uuid.uuid4())

# Perform fleet provisioning. If it's successful, a newly created thing should appear.
test_result = run_in_ci.setup_and_launch(cfg_file, thing_uuid)

# Delete a thing created by fleet provisioning.
# NOTE We want to try to delete thing even if test was unsuccessful.
thing_name = parsed_commands.thing_name_prefix + thing_uuid
delete_result = ci_iot_thing.delete_iot_thing(thing_name, parsed_commands.region)

if test_result != 0 or delete_result != 0:
sys.exit(-1)

if __name__ == "__main__":
main()

0 comments on commit e1f8308

Please sign in to comment.