fix: convert ip_filter values into the target type #365

Workflow file for this run

.github/workflows/incremental-acceptance-tests.yml at 661e8c8

	name: Incremental Acceptance Tests

	on:
	pull_request:
	branches:
	- main
	- v*
	types:
	- opened
	- synchronize
	- reopened
	- labeled
	- unlabeled
	push:
	branches:
	- main
	- v*
	workflow_dispatch: {}

	permissions: read-all

	concurrency:
	group: ci-${{ github.head_ref \|\| github.ref }}
	cancel-in-progress: true

	jobs:
	selproj:
	name: Run selproj
	runs-on: ubuntu-latest
	if: >
	(github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'skip workflows')) \|\|
	github.event_name == 'push'

	outputs:
	suffix: ${{ steps.selproj.outputs.suffix }}

	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- name: Set up Go
	uses: actions/setup-go@v5
	with:
	go-version-file: go.mod

	- name: Run selproj
	id: selproj
	run: echo "suffix=$(make -s ci-selproj \| tr -d '\n')" >> $GITHUB_OUTPUT
	env:
	AIVEN_TOKEN: ${{ secrets.AIVEN_TOKEN }}
	AIVEN_PROJECT_NAME_PREFIX: ${{ secrets.AIVEN_PROJECT_NAME_PREFIX }}

	find_sdkv2_entities:
	name: Find SDKv2 entities
	runs-on: ubuntu-latest
	if: >
	(github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'skip workflows')) \|\|
	github.event_name == 'push'

	env:
	SDKV2_IMPL_PATH: internal/sdkprovider

	outputs:
	entities: ${{ steps.find_entities.outputs.entities }}

	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- name: Find provider.go file
	id: find_provider_go_file
	run: \|
	# Find the path to 'provider.go' within the specified directory and append it to the output
	echo "path=$(find "$SDKV2_IMPL_PATH" -type f -name 'provider.go')" >> $GITHUB_OUTPUT

	- name: Extract imports
	id: extract_imports
	run: \|
	# Capture the path of the Go file from a previous step's output
	provider_go_file=${{ steps.find_provider_go_file.outputs.path }}

	# Extract the import block from the Go file using awk
	imports=$(awk '/^import $/{flag=1;next}/^$/{flag=0}flag' "$provider_go_file")

	# Format the imports to a JSON array, removing spaces and quotes
	imports=$(echo "$imports" \| jq -R -s -c 'split("\n") \| map(select(length > 0) \| gsub("[[:space:]\"]";""))')

	# Append the formatted imports to the output
	echo "imports=$imports" >> $GITHUB_OUTPUT

	- name: Build import map
	id: build_import_map
	run: \|
	# Extract the imports from the previous step's output and read line by line
	import_lines=$(echo '${{ steps.extract_imports.outputs.imports }}' \| jq -r '.[]')
	import_map="{}"

	# Process each line to extract package information
	while IFS= read -r line; do
	# Check if the line matches the expected format with SDKV2_IMPL_PATH
	if [[ $line =~ ^(.)"$SDKV2_IMPL_PATH"/(.)$ ]]; then
	package_path="${BASH_REMATCH[2]}"
	package_name=$(basename "$package_path")

	# Log the found package name and its path
	echo "Found package $package_name in $SDKV2_IMPL_PATH/$package_path"

	# Update the import_map with the new package
	import_map=$(
	echo "$import_map" \| jq -c --arg key "$package_name" --arg value "$package_path" \
	'. + {($key): $value}'
	)
	fi
	done <<< "$import_lines"

	# Store the final import_map in the output
	echo "import_map=$import_map" >> $GITHUB_OUTPUT

	- name: Find entities
	id: find_entities
	run: \|
	# Set up variables using outputs from previous steps
	provider_go_file=${{ steps.find_provider_go_file.outputs.path }}
	import_map='${{ steps.build_import_map.outputs.import_map }}'
	entities="[]"

	# Read each line from the awk command output
	while read -r name type func_with_parentheses; do
	# Skip if name or func_with_parentheses is empty
	if [[ -z "$name" \|\| -z "$func_with_parentheses" ]]; then
	continue
	fi

	# Extract function name and package name from func_with_parentheses
	func="${func_with_parentheses%)*}"
	func="${func##*.}"
	package_name="${func_with_parentheses%%.*}"
	# Find the path using the package name from the import map
	path=$(echo $import_map \| jq -r --arg package_name "$package_name" '.[$package_name] // empty')

	# If a path is found, proceed to find the .go file containing the function
	if [[ -n "$path" ]]; then
	path="$SDKV2_IMPL_PATH/$path"
	file=$(find "$path" -type f -name '*.go' -exec grep -l "$func" {} +)

	# If a file is found, add the entity to the entities list
	if [[ -n "$file" ]]; then
	echo "Found $type $name in $file"
	entities=$(
	echo "$entities" \| jq -c --arg name "$name" --arg type "$type" --arg file "$file" \
	'. + [{($name): {("type"): $type, ("file"): $file}}]'
	)
	fi
	fi
	done < <(awk -v file="$provider_go_file" '
	# Begin capturing when encountering ResourcesMap or DataSourcesMap
	/ResourcesMap: map\[string\]\*schema\.Resource{/ { type="resource"; capture = 1; next }
	/DataSourcesMap: map\[string\]\*schema\.Resource{/ { type="datasource"; capture = 1; next }
	# Stop capturing after a closing brace
	capture && /\}/ { capture = 0 }
	# For each captured line, parse and print the name, type, and function
	capture && NF {
	gsub(/[\t "]+/, "")
	split($0, a, ":")
	print a[1], type, a[2]
	}
	' "$provider_go_file")

	# Output the entities to the output
	echo "entities=$entities" >> $GITHUB_OUTPUT

	find_plugin_framework_entities:
	name: Find Plugin Framework entities
	runs-on: ubuntu-latest
	if: >
	(github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'skip workflows')) \|\|
	github.event_name == 'push'

	env:
	PLUGIN_FRAMEWORK_IMPL_PATH: internal/plugin

	outputs:
	entities: ${{ steps.find_entities.outputs.entities }}

	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- name: Find entities
	id: find_entities
	run: \|
	entities="[]" # Initialize an empty JSON array to hold entity details

	# Find all .go files in the specified directory and its subdirectories
	while IFS= read -r file; do
	# Extract relevant lines from the file and parse them
	while IFS=, read -r name type file; do
	echo "Found $type $name in $file" # Log the found entities
	# Append the found entity to the entities JSON array
	entities=$(
	echo "$entities" \| jq -c --arg name "$name" --arg type "$type" --arg file "$file" \
	'. + [{($name): {("type"): $type, ("file"): $file}}]'
	)
	done < <(awk -v file="$file" '
	/resp\.TypeName\s=\sreq\.ProviderTypeName\s\+\s"/ {
	match($0, /"(_[^"]+)"/, arr)
	if (length(arr[1]) > 0) {
	type = "datasource"
	# Check for specific patterns to determine if the entity is a resource
	while((getline line < file) > 0) {
	if (line ~ /_ resource\.Resource/) {
	type = "resource"
	break
	}
	}
	close(file)
	# Output the entity name, type, and file in CSV format
	print "aiven" arr[1] "," type "," file
	}
	}
	' "$file")
	done < <(find "$PLUGIN_FRAMEWORK_IMPL_PATH" -type f -name '*.go')

	echo "entities=$entities" >> $GITHUB_OUTPUT # Output the final entities JSON array to the output

	combine_entities:
	name: Combine entities
	runs-on: ubuntu-latest

	needs:
	- find_sdkv2_entities
	- find_plugin_framework_entities

	outputs:
	entities: ${{ steps.combine_entities.outputs.entities }}

	steps:
	- name: Combine entities
	id: combine_entities
	run: \|
	# Initialize an empty JSON array to store entities
	entities='[]'

	# Read SDKv2 entities from a previous step's output and convert them into an array
	mapfile -t sdkv2_entities < <(echo '${{ needs.find_sdkv2_entities.outputs.entities }}' \| jq -c '.[]')

	# Read plugin framework entities from a previous step's output and convert them into an array
	mapfile -t plugin_framework_entities < <(
	echo '${{ needs.find_plugin_framework_entities.outputs.entities }}' \| jq -c '.[]'
	)

	# Define a function to add an entity to the 'entities' JSON array
	add_entity_to_entities() {
	local entity_json=$1
	entities=$(
	jq -c --argjson entities "$entities" --argjson entity_json "$entity_json" \
	'$entities + [($entity_json \| to_entries \| .[0] \| {key: .key, value: .value}) \| {(.key): .value}]' \
	<<< "{}"
	)
	}

	# Iterate over all entities and add them to the 'entities' JSON array
	for entity_json in "${sdkv2_entities[@]}" "${plugin_framework_entities[@]}"; do
	add_entity_to_entities "$entity_json"
	done

	# Output the 'entities' JSON array
	echo "entities=$entities" >> $GITHUB_OUTPUT

	find_tests:
	name: Find tests
	runs-on: ubuntu-latest

	needs:
	- combine_entities

	outputs:
	tests: ${{ steps.find_tests.outputs.tests }}

	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- name: Set up Go
	uses: actions/setup-go@v5
	with:
	go-version-file: go.mod

	- name: Find tests
	id: find_tests
	run: \|
	# Collect changed test files that match a specific pattern
	mapfile -t changed_test_files < <(
	git diff "origin/${{ github.base_ref \|\| 'main' }}" --name-only \| grep '_test\.go$'
	)
	# Collect entities from previous workflow step
	mapfile -t entities < <(echo '${{ needs.combine_entities.outputs.entities }}' \| jq -c '.[]')
	# Declare an associative array to keep track of entity tests
	declare -A entity_tests

	# Iterate over changed test files to find related entities
	for test_file in "${changed_test_files[@]}"; do
	while IFS= read -r line; do
	# Check if the line defines a resource or data entity
	if [[ $line =~ resource\ \"([^\"]+)\" \|\| $line =~ data\ \"([^\"]+)\" ]]; then
	entity="${BASH_REMATCH[1]}"
	echo "Found entity $entity in changed test file $test_file"
	# Add the test file to the entity_tests array for the detected entity
	entity_tests[$entity]+="$test_file "
	fi
	done < "$test_file"
	done

	# Iterate over entities to check for changes and dependencies
	for entity in "${entities[@]}"; do
	name=$(echo "$entity" \| jq -r 'keys[0]')
	type=$(echo "$entity" \| jq -r --arg name "$name" '.[$name].type')
	file=$(echo "$entity" \| jq -r --arg name "$name" '.[$name].file')
	dir=$(dirname "$file")

	# Check if the entity definition file has changed
	file_changed=$(
	git diff "origin/${{ github.base_ref \|\| 'main' }}" --name-only \|
	grep -q "^$file$" && echo true \|\| echo false
	)
	dep_changed=false

	# Check if there are changes in tests related to the entity
	if [[ ${entity_tests[$name]+_} ]]; then
	echo "Test involving $name $type was changed"
	elif [[ $file_changed == true ]]; then
	echo "Definition of $name $type was changed"
	else
	# Construct the repository URL
	repo_url="${{ github.server_url }}/${{ github.repository }}"
	repo_url="${repo_url/https:\/\/}"
	# List dependencies of the entity and check for changes
	deps=$(go list -json "./$dir" \| jq -r '.Deps[]' \| grep -oP "^$repo_url/\K.*")

	for dep in $deps; do
	if git diff "origin/${{ github.base_ref \|\| 'main' }}" --name-only \| grep -q "$dep"; then
	echo "Dependency $dep of $name $type changed"
	dep_changed=true
	break
	fi
	done
	fi

	# If there are changes in the file, dependencies, or related tests, find all relevant test files
	if [[ $file_changed == true \|\| $dep_changed == true \|\| ${entity_tests[$name]+_} ]]; then
	pattern=$([[ "$type" == "resource" ]] && echo "resource \"$name\"" \|\| echo "data \"$name\"")
	while IFS= read -r test_file; do
	echo "Found test for $name $type in $test_file"
	entity_tests[$name]+="$test_file "
	done < <(find internal -type f -name '*_test.go' -exec grep -l "$pattern" {} +)
	fi
	done

	# Compile a unique list of test files to run
	tests=()
	for test_files in "${entity_tests[@]}"; do
	for test_file in $test_files; do
	[[ ! " ${tests[*]} " =~ " ${test_file} " ]] && tests+=("$test_file")
	done
	done

	# Convert the list of tests into a JSON array and add it to the output
	tests_json=$(printf '%s\n' "${tests[@]}" \| jq -R . \| jq -s -c .)
	echo "tests=$tests_json" >> $GITHUB_OUTPUT

	run_tests:
	name: Run tests
	runs-on: ubuntu-latest
	if: needs.find_tests.outputs.tests != '[""]'

	needs:
	- selproj
	- find_tests

	env:
	COUNT: 1
	PARALLEL: 10
	TIMEOUT: 180m

	strategy:
	max-parallel: 5
	matrix:
	test: ${{ fromJson(needs.find_tests.outputs.tests) }}

	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- name: Set up Go
	uses: actions/setup-go@v5
	with:
	go-version-file: go.mod

	- name: Run tests
	run: >
	go test ./$(dirname '${{ matrix.test }}') -v -count 1 -parallel 10 -timeout 180m -run
	$(grep -oP 'func \KTest\w+' ${{ matrix.test }} \| xargs \| sed 's/ /\|/g')
	env:
	TF_ACC: 1
	PROVIDER_AIVEN_ENABLE_BETA: 1
	CGO_ENABLED: 0
	AIVEN_TOKEN: ${{ secrets.AIVEN_TOKEN }}
	AIVEN_PROJECT_NAME: ${{ secrets.AIVEN_PROJECT_NAME_PREFIX }}${{ needs.selproj.outputs.suffix }}
	AIVEN_ORGANIZATION_NAME: ${{ secrets.AIVEN_ORGANIZATION_NAME }}
	AIVEN_ACCOUNT_NAME: ${{ secrets.AIVEN_ORGANIZATION_NAME }}

	result:
	# Because we use a matrix strategy, we need to check the results of all jobs, and if any of them failed or were
	# cancelled, we should fail the workflow. This is used to set up a status check on the pull request.

	name: Result
	runs-on: ubuntu-latest
	if: always()

	needs:
	- run_tests

	steps:
	- if: ${{ contains(needs..result, 'failure') \|\| contains(needs..result, 'cancelled') }}
	run: exit 1

	sweep:
	name: Sweep
	runs-on: ubuntu-latest
	if: always()

	needs:
	- selproj
	- result

	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- name: Set up Go
	uses: actions/setup-go@v5
	with:
	go-version-file: go.mod

	- name: Sweep
	uses: nick-invision/retry@v3
	with:
	timeout_minutes: 15
	max_attempts: 15
	command: make sweep
	env:
	AIVEN_TOKEN: ${{ secrets.AIVEN_TOKEN }}
	AIVEN_PROJECT_NAME: ${{ secrets.AIVEN_PROJECT_NAME_PREFIX }}${{ needs.selproj.outputs.suffix }}
	AIVEN_ORGANIZATION_NAME: ${{ secrets.AIVEN_ORGANIZATION_NAME }}
	AIVEN_ACCOUNT_NAME: ${{ secrets.AIVEN_ORGANIZATION_NAME }}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

fix: convert ip_filter values into the target type #365

Workflow file

fix: convert ip_filter values into the target type #365

Jobs

Run details

Workflow file for this run