diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bbca43a..aebda51 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,24 +5,25 @@ env: jobs: test: name: Test - strategy: - matrix: - pg-version: ["12", "13"] runs-on: ubuntu-latest + + services: + postgres: + image: postgis/postgis:14-3.3-alpine + env: + POSTGRES_DB: archiver_test + POSTGRES_USER: archiver_test + POSTGRES_PASSWORD: temba + ports: + - 5432:5432 + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + steps: - name: Checkout code - uses: actions/checkout@v1 - - - name: Install PostgreSQL - uses: harmon758/postgresql-action@v1 - with: - postgresql version: ${{ matrix.pg-version }} - postgresql db: archiver_test - postgresql user: temba - postgresql password: temba + uses: actions/checkout@v3 - name: Install Go - uses: actions/setup-go@v1 + uses: actions/setup-go@v3 with: go-version: ${{ env.go-version }} @@ -34,7 +35,7 @@ jobs: - name: Upload coverage if: success() - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v3 with: token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: true @@ -46,18 +47,19 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v1 + uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Install Go - uses: actions/setup-go@v1 + uses: actions/setup-go@v3 with: go-version: ${{ env.go-version }} - name: Publish release - uses: goreleaser/goreleaser-action@v1 + uses: goreleaser/goreleaser-action@v4 with: - version: v0.147.2 - args: release --rm-dist + args: release --clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} fail_ci_if_error: true diff --git a/.gitignore b/.gitignore index a40925b..1f048fd 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,7 @@ *~ .vscode dist/ - +*.DS_store *.py *.pyc diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a0b858..3107aad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,47 @@ +v8.2.0 (2023-07-31) +------------------------- + * Update .gitignore + +v8.1.7 (2023-06-06) +------------------------- + * Fix release CHANGELOG generation + +v8.1.6 (2023-06-06) +------------------------- + * Remove deleting of channel logs as these are no longer linked to messages + +v8.1.5 (2023-03-24) +------------------------- + * Revert to go 1.19 + +v8.1.4 (2023-03-15) +------------------------- + * Match API and always return type=text|voice for messages + +v8.1.3 (2023-03-09) +------------------------- + * Update dependencies and use go 1.20 + * Update test database schema and cleanup sql queries + +v8.1.2 (2023-02-20) +------------------------- + * Add support for msg_type = T + +v8.1.1 (2023-02-15) +------------------------- + * Don't try to delete broadcast URNs which no longer exist + +v8.1.0 (2023-01-18) +------------------------- + * Delete old flow starts after deleting runs + +v8.0.0 (2023-01-09) +------------------------- + * Only fetch broadcasts which don't have messages + * Remove use of deprecated ioutil package + * Update testdb.sql to reflect schema changes and cleanup sql variables + * Test against postgres 14 + v7.5.0 ---------- * Use go 1.19 diff --git a/LICENSE b/LICENSE index 109db71..d506834 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ -Archiver - content archival service for the TextIt platform. +Archiver - content archival service for RapidPro/TextIt. -Copyright (C) 2018-2022 TextIt. +Copyright (C) 2018-2023 Nyaruka, Inc. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as diff --git a/archives/archives.go b/archives/archives.go index 54c6274..e906dc4 100644 --- a/archives/archives.go +++ b/archives/archives.go @@ -9,7 +9,6 @@ import ( "encoding/hex" "fmt" "io" - "io/ioutil" "os" "path/filepath" "time" @@ -92,18 +91,18 @@ func (a *Archive) endDate() time.Time { return endDate } -const lookupActiveOrgs = ` -SELECT o.id, o.name, o.created_on, o.is_anon -FROM orgs_org o -WHERE o.is_active = TRUE order by o.id -` +const sqlLookupActiveOrgs = ` + SELECT id, name, created_on, is_anon + FROM orgs_org + WHERE is_active +ORDER BY id` // GetActiveOrgs returns the active organizations sorted by id func GetActiveOrgs(ctx context.Context, db *sqlx.DB, conf *Config) ([]Org, error) { ctx, cancel := context.WithTimeout(ctx, time.Minute) defer cancel() - rows, err := db.QueryxContext(ctx, lookupActiveOrgs) + rows, err := db.QueryxContext(ctx, sqlLookupActiveOrgs) if err != nil { return nil, errors.Wrapf(err, "error fetching active orgs") } @@ -122,11 +121,11 @@ func GetActiveOrgs(ctx context.Context, db *sqlx.DB, conf *Config) ([]Org, error return orgs, nil } -const lookupOrgArchives = ` -SELECT id, org_id, start_date::timestamp with time zone as start_date, period, archive_type, hash, size, record_count, url, rollup_id, needs_deletion -FROM archives_archive WHERE org_id = $1 AND archive_type = $2 -ORDER BY start_date asc, period desc -` +const sqlLookupOrgArchives = ` + SELECT id, org_id, start_date::timestamp with time zone AS start_date, period, archive_type, hash, size, record_count, url, rollup_id, needs_deletion + FROM archives_archive + WHERE org_id = $1 AND archive_type = $2 +ORDER BY start_date ASC, period DESC` // GetCurrentArchives returns all the current archives for the passed in org and record type func GetCurrentArchives(ctx context.Context, db *sqlx.DB, org Org, archiveType ArchiveType) ([]*Archive, error) { @@ -134,7 +133,7 @@ func GetCurrentArchives(ctx context.Context, db *sqlx.DB, org Org, archiveType A defer cancel() archives := make([]*Archive, 0, 1) - err := db.SelectContext(ctx, &archives, lookupOrgArchives, org.ID, archiveType) + err := db.SelectContext(ctx, &archives, sqlLookupOrgArchives, org.ID, archiveType) if err != nil && err != sql.ErrNoRows { return nil, errors.Wrapf(err, "error selecting current archives for org: %d and type: %s", org.ID, archiveType) } @@ -142,11 +141,11 @@ func GetCurrentArchives(ctx context.Context, db *sqlx.DB, org Org, archiveType A return archives, nil } -const lookupArchivesNeedingDeletion = ` -SELECT id, org_id, start_date::timestamp with time zone as start_date, period, archive_type, hash, size, record_count, url, rollup_id, needs_deletion -FROM archives_archive WHERE org_id = $1 AND archive_type = $2 AND needs_deletion = TRUE -ORDER BY start_date asc, period desc -` +const sqlLookupArchivesNeedingDeletion = ` + SELECT id, org_id, start_date::timestamp with time zone AS start_date, period, archive_type, hash, size, record_count, url, rollup_id, needs_deletion + FROM archives_archive + WHERE org_id = $1 AND archive_type = $2 AND needs_deletion = TRUE +ORDER BY start_date ASC, period DESC` // GetArchivesNeedingDeletion returns all the archives which need to be deleted func GetArchivesNeedingDeletion(ctx context.Context, db *sqlx.DB, org Org, archiveType ArchiveType) ([]*Archive, error) { @@ -154,7 +153,7 @@ func GetArchivesNeedingDeletion(ctx context.Context, db *sqlx.DB, org Org, archi defer cancel() archives := make([]*Archive, 0, 1) - err := db.SelectContext(ctx, &archives, lookupArchivesNeedingDeletion, org.ID, archiveType) + err := db.SelectContext(ctx, &archives, sqlLookupArchivesNeedingDeletion, org.ID, archiveType) if err != nil && err != sql.ErrNoRows { return nil, errors.Wrapf(err, "error selecting archives needing deletion for org: %d and type: %s", org.ID, archiveType) } @@ -162,11 +161,10 @@ func GetArchivesNeedingDeletion(ctx context.Context, db *sqlx.DB, org Org, archi return archives, nil } -const lookupCountOrgArchives = ` +const sqlCountOrgArchives = ` SELECT count(id) -FROM archives_archive -WHERE org_id = $1 AND archive_type = $2 -` + FROM archives_archive + WHERE org_id = $1 AND archive_type = $2` // GetCurrentArchiveCount returns the archive count for the passed in org and record type func GetCurrentArchiveCount(ctx context.Context, db *sqlx.DB, org Org, archiveType ArchiveType) (int, error) { @@ -175,7 +173,7 @@ func GetCurrentArchiveCount(ctx context.Context, db *sqlx.DB, org Org, archiveTy var archiveCount int - err := db.GetContext(ctx, &archiveCount, lookupCountOrgArchives, org.ID, archiveType) + err := db.GetContext(ctx, &archiveCount, sqlCountOrgArchives, org.ID, archiveType) if err != nil { return 0, errors.Wrapf(err, "error querying archive count for org: %d and type: %s", org.ID, archiveType) } @@ -184,12 +182,11 @@ func GetCurrentArchiveCount(ctx context.Context, db *sqlx.DB, org Org, archiveTy } // between is inclusive on both sides -const lookupOrgDailyArchivesForDateRange = ` -SELECT id, start_date::timestamp with time zone as start_date, period, archive_type, hash, size, record_count, url, rollup_id -FROM archives_archive -WHERE org_id = $1 AND archive_type = $2 AND period = $3 AND start_date BETWEEN $4 AND $5 -ORDER BY start_date asc -` +const sqlLookupOrgDailyArchivesForDateRange = ` + SELECT id, start_date::timestamp with time zone AS start_date, period, archive_type, hash, size, record_count, url, rollup_id + FROM archives_archive + WHERE org_id = $1 AND archive_type = $2 AND period = $3 AND start_date BETWEEN $4 AND $5 +ORDER BY start_date ASC` // GetDailyArchivesForDateRange returns all the current archives for the passed in org and record type and date range func GetDailyArchivesForDateRange(ctx context.Context, db *sqlx.DB, org Org, archiveType ArchiveType, startDate time.Time, endDate time.Time) ([]*Archive, error) { @@ -198,7 +195,7 @@ func GetDailyArchivesForDateRange(ctx context.Context, db *sqlx.DB, org Org, arc existingArchives := make([]*Archive, 0, 1) - err := db.SelectContext(ctx, &existingArchives, lookupOrgDailyArchivesForDateRange, org.ID, archiveType, DayPeriod, startDate, endDate) + err := db.SelectContext(ctx, &existingArchives, sqlLookupOrgDailyArchivesForDateRange, org.ID, archiveType, DayPeriod, startDate, endDate) if err != nil && err != sql.ErrNoRows { return nil, errors.Wrapf(err, "error selecting daily archives for org: %d and type: %s", org.ID, archiveType) } @@ -219,7 +216,7 @@ func GetMissingDailyArchives(ctx context.Context, db *sqlx.DB, now time.Time, or return GetMissingDailyArchivesForDateRange(ctx, db, startDate, endDate, org, archiveType) } -const lookupMissingDailyArchive = ` +const sqlLookupMissingDailyArchive = ` WITH month_days(missing_day) AS ( select GENERATE_SERIES($1::timestamp with time zone, $2::timestamp with time zone, '1 day')::date ), curr_archives AS ( @@ -227,11 +224,13 @@ WITH month_days(missing_day) AS ( UNION DISTINCT -- also get the overlapping days for the monthly rolled up archives SELECT GENERATE_SERIES(start_date, (start_date + '1 month'::interval) - '1 second'::interval, '1 day')::date AS start_date - FROM archives_archive WHERE org_id = $3 AND period = 'M' AND archive_type=$5 + FROM archives_archive + WHERE org_id = $3 AND period = 'M' AND archive_type = $5 ) -SELECT missing_day::timestamp WITH TIME ZONE FROM month_days LEFT JOIN curr_archives ON curr_archives.start_date = month_days.missing_day -WHERE curr_archives.start_date IS NULL -` + SELECT missing_day::timestamp with time zone + FROM month_days +LEFT JOIN curr_archives ON curr_archives.start_date = month_days.missing_day + WHERE curr_archives.start_date IS NULL` // GetMissingDailyArchivesForDateRange returns all them missing daily archives between the two passed in date ranges func GetMissingDailyArchivesForDateRange(ctx context.Context, db *sqlx.DB, startDate time.Time, endDate time.Time, org Org, archiveType ArchiveType) ([]*Archive, error) { @@ -240,7 +239,7 @@ func GetMissingDailyArchivesForDateRange(ctx context.Context, db *sqlx.DB, start missing := make([]*Archive, 0, 1) - rows, err := db.QueryxContext(ctx, lookupMissingDailyArchive, startDate, endDate, org.ID, DayPeriod, archiveType) + rows, err := db.QueryxContext(ctx, sqlLookupMissingDailyArchive, startDate, endDate, org.ID, DayPeriod, archiveType) if err != nil { return nil, errors.Wrapf(err, "error getting missing daily archives for org: %d and type: %s", org.ID, archiveType) } @@ -269,14 +268,16 @@ func GetMissingDailyArchivesForDateRange(ctx context.Context, db *sqlx.DB, start // startDate is truncated to the first of the month // endDate for range is not inclusive so we must deduct 1 second -const lookupMissingMonthlyArchive = ` +const sqlLookupMissingMonthlyArchive = ` WITH month_days(missing_month) AS ( SELECT generate_series(date_trunc('month', $1::timestamp with time zone), $2::timestamp with time zone - '1 second'::interval, '1 month')::date ), curr_archives AS ( - SELECT start_date FROM archives_archive WHERE org_id = $3 and period = $4 and archive_type=$5 + SELECT start_date FROM archives_archive WHERE org_id = $3 and period = $4 and archive_type = $5 ) -SELECT missing_month::timestamp with time zone from month_days LEFT JOIN curr_archives ON curr_archives.start_date = month_days.missing_month -WHERE curr_archives.start_date IS NULL + SELECT missing_month::timestamp with time zone + FROM month_days +LEFT JOIN curr_archives ON curr_archives.start_date = month_days.missing_month + WHERE curr_archives.start_date IS NULL ` // GetMissingMonthlyArchives gets which montly archives are currently missing for this org @@ -292,7 +293,7 @@ func GetMissingMonthlyArchives(ctx context.Context, db *sqlx.DB, now time.Time, missing := make([]*Archive, 0, 1) - rows, err := db.QueryxContext(ctx, lookupMissingMonthlyArchive, startDate, endDate, org.ID, MonthPeriod, archiveType) + rows, err := db.QueryxContext(ctx, sqlLookupMissingMonthlyArchive, startDate, endDate, org.ID, MonthPeriod, archiveType) if err != nil { return nil, errors.Wrapf(err, "error getting missing monthly archive for org: %d and type: %s", org.ID, archiveType) } @@ -346,7 +347,7 @@ func BuildRollupArchive(ctx context.Context, db *sqlx.DB, conf *Config, s3Client // great, we have all the dailies we need, download them filename := fmt.Sprintf("%s_%d_%s_%d_%02d_", monthlyArchive.ArchiveType, monthlyArchive.Org.ID, monthlyArchive.Period, monthlyArchive.StartDate.Year(), monthlyArchive.StartDate.Month()) - file, err := ioutil.TempFile(conf.TempDir, filename) + file, err := os.CreateTemp(conf.TempDir, filename) if err != nil { return errors.Wrapf(err, "error creating temp file: %s", filename) } @@ -479,7 +480,7 @@ func CreateArchiveFile(ctx context.Context, db *sqlx.DB, archive *Archive, archi }) filename := fmt.Sprintf("%s_%d_%s%d%02d%02d_", archive.ArchiveType, archive.Org.ID, archive.Period, archive.StartDate.Year(), archive.StartDate.Month(), archive.StartDate.Day()) - file, err := ioutil.TempFile(archivePath, filename) + file, err := os.CreateTemp(archivePath, filename) if err != nil { return errors.Wrapf(err, "error creating temp file: %s", filename) } @@ -594,11 +595,10 @@ func UploadArchive(ctx context.Context, s3Client s3iface.S3API, bucket string, a return nil } -const insertArchive = ` +const sqlInsertArchive = ` INSERT INTO archives_archive(archive_type, org_id, created_on, start_date, period, record_count, size, hash, url, needs_deletion, build_time, rollup_id) -VALUES(:archive_type, :org_id, :created_on, :start_date, :period, :record_count, :size, :hash, :url, :needs_deletion, :build_time, :rollup_id) -RETURNING id -` + VALUES(:archive_type, :org_id, :created_on, :start_date, :period, :record_count, :size, :hash, :url, :needs_deletion, :build_time, :rollup_id) + RETURNING id` // WriteArchiveToDB write an archive to the Database func WriteArchiveToDB(ctx context.Context, db *sqlx.DB, archive *Archive) error { @@ -613,7 +613,7 @@ func WriteArchiveToDB(ctx context.Context, db *sqlx.DB, archive *Archive) error return errors.Wrapf(err, "error starting transaction") } - rows, err := tx.NamedQuery(insertArchive, archive) + rows, err := tx.NamedQuery(sqlInsertArchive, archive) if err != nil { tx.Rollback() return errors.Wrapf(err, "error inserting archive") @@ -828,11 +828,7 @@ func RollupOrgArchives(ctx context.Context, now time.Time, config *Config, db *s return created, failed, nil } -const setArchiveDeleted = ` -UPDATE archives_archive -SET needs_deletion = FALSE, deleted_on = $2 -WHERE id = $1 -` +const sqlUpdateArchiveDeleted = `UPDATE archives_archive SET needs_deletion = FALSE, deleted_on = $2 WHERE id = $1` var deleteTransactionSize = 100 @@ -867,6 +863,10 @@ func DeleteArchivedOrgRecords(ctx context.Context, now time.Time, config *Config case RunType: err = DeleteArchivedRuns(ctx, config, db, s3Client, a) + if err == nil { + err = DeleteFlowStarts(ctx, now, config, db, org) + } + default: err = fmt.Errorf("unknown archive type: %s", a.ArchiveType) } diff --git a/archives/archives_test.go b/archives/archives_test.go index 573f344..f155b1b 100644 --- a/archives/archives_test.go +++ b/archives/archives_test.go @@ -3,7 +3,7 @@ package archives import ( "compress/gzip" "context" - "io/ioutil" + "io" "os" "testing" "time" @@ -18,10 +18,10 @@ import ( ) func setup(t *testing.T) *sqlx.DB { - testDB, err := ioutil.ReadFile("../testdb.sql") + testDB, err := os.ReadFile("../testdb.sql") assert.NoError(t, err) - db, err := sqlx.Open("postgres", "postgres://temba:temba@localhost:5432/archiver_test?sslmode=disable&TimeZone=UTC") + db, err := sqlx.Open("postgres", "postgres://archiver_test:temba@localhost:5432/archiver_test?sslmode=disable&TimeZone=UTC") assert.NoError(t, err) _, err = db.Exec(string(testDB)) @@ -40,25 +40,26 @@ func TestGetMissingDayArchives(t *testing.T) { orgs, err := GetActiveOrgs(ctx, db, config) assert.NoError(t, err) + assert.Len(t, orgs, 3) now := time.Date(2018, 1, 8, 12, 30, 0, 0, time.UTC) // org 1 is too new, no tasks tasks, err := GetMissingDailyArchives(ctx, db, now, orgs[0], MessageType) assert.NoError(t, err) - assert.Equal(t, 0, len(tasks)) + assert.Len(t, tasks, 0) // org 2 should have some tasks, err = GetMissingDailyArchives(ctx, db, now, orgs[1], MessageType) assert.NoError(t, err) - assert.Equal(t, 61, len(tasks)) + assert.Len(t, tasks, 61) assert.Equal(t, time.Date(2017, 8, 10, 0, 0, 0, 0, time.UTC), tasks[0].StartDate) assert.Equal(t, time.Date(2017, 10, 10, 0, 0, 0, 0, time.UTC), tasks[60].StartDate) // org 3 is the same as 2, but two of the tasks have already been built tasks, err = GetMissingDailyArchives(ctx, db, now, orgs[2], MessageType) assert.NoError(t, err) - assert.Equal(t, 31, len(tasks)) + assert.Len(t, tasks, 31) assert.Equal(t, time.Date(2017, 8, 11, 0, 0, 0, 0, time.UTC), tasks[0].StartDate) assert.Equal(t, time.Date(2017, 10, 1, 0, 0, 0, 0, time.UTC), tasks[21].StartDate) assert.Equal(t, time.Date(2017, 10, 10, 0, 0, 0, 0, time.UTC), tasks[30].StartDate) @@ -67,13 +68,13 @@ func TestGetMissingDayArchives(t *testing.T) { orgs[2].RetentionPeriod = 200 tasks, err = GetMissingDailyArchives(ctx, db, now, orgs[2], MessageType) assert.NoError(t, err) - assert.Equal(t, 0, len(tasks)) + assert.Len(t, tasks, 0) // org 1 again, but lowering the archive period so we have tasks orgs[0].RetentionPeriod = 2 tasks, err = GetMissingDailyArchives(ctx, db, now, orgs[0], MessageType) assert.NoError(t, err) - assert.Equal(t, 58, len(tasks)) + assert.Len(t, tasks, 58) assert.Equal(t, time.Date(2017, 11, 10, 0, 0, 0, 0, time.UTC), tasks[0].StartDate) assert.Equal(t, time.Date(2017, 12, 1, 0, 0, 0, 0, time.UTC), tasks[21].StartDate) assert.Equal(t, time.Date(2017, 12, 10, 0, 0, 0, 0, time.UTC), tasks[30].StartDate) @@ -147,9 +148,9 @@ func TestCreateMsgArchive(t *testing.T) { // should have two records, second will have attachments assert.Equal(t, 3, task.RecordCount) - assert.Equal(t, int64(528), task.Size) + assert.Equal(t, int64(522), task.Size) assert.Equal(t, time.Date(2017, 8, 12, 0, 0, 0, 0, time.UTC), task.StartDate) - assert.Equal(t, "b3bf00bf1234ea47f14ffd0171a8ead0", task.Hash) + assert.Equal(t, "c2c12d94eb758a3c06c5c4e0706934ff", task.Hash) assertArchiveFile(t, task, "messages1.jsonl") DeleteArchiveFile(task) @@ -167,8 +168,8 @@ func TestCreateMsgArchive(t *testing.T) { // should have one record assert.Equal(t, 1, task.RecordCount) - assert.Equal(t, int64(294), task.Size) - assert.Equal(t, "bd163ead077774425aa559e30d48ca87", task.Hash) + assert.Equal(t, int64(293), task.Size) + assert.Equal(t, "c8245a44279102a1612170df3787c32d", task.Hash) assertArchiveFile(t, task, "messages2.jsonl") DeleteArchiveFile(task) @@ -180,10 +181,10 @@ func assertArchiveFile(t *testing.T, archive *Archive, truthName string) { zTestReader, err := gzip.NewReader(testFile) assert.NoError(t, err) - test, err := ioutil.ReadAll(zTestReader) + test, err := io.ReadAll(zTestReader) assert.NoError(t, err) - truth, err := ioutil.ReadFile("./testdata/" + truthName) + truth, err := os.ReadFile("./testdata/" + truthName) assert.NoError(t, err) assert.Equal(t, truth, test) @@ -332,14 +333,14 @@ func TestArchiveOrgMessages(t *testing.T) { assert.Equal(t, 61, len(dailiesCreated)) assertArchive(t, dailiesCreated[0], time.Date(2017, 8, 10, 0, 0, 0, 0, time.UTC), DayPeriod, 0, 23, "f0d79988b7772c003d04a28bd7417a62") assertArchive(t, dailiesCreated[1], time.Date(2017, 8, 11, 0, 0, 0, 0, time.UTC), DayPeriod, 0, 23, "f0d79988b7772c003d04a28bd7417a62") - assertArchive(t, dailiesCreated[2], time.Date(2017, 8, 12, 0, 0, 0, 0, time.UTC), DayPeriod, 3, 528, "b3bf00bf1234ea47f14ffd0171a8ead0") - assertArchive(t, dailiesCreated[3], time.Date(2017, 8, 13, 0, 0, 0, 0, time.UTC), DayPeriod, 1, 312, "32e61b1431217b59fca0170f637d78a3") + assertArchive(t, dailiesCreated[2], time.Date(2017, 8, 12, 0, 0, 0, 0, time.UTC), DayPeriod, 3, 522, "c2c12d94eb758a3c06c5c4e0706934ff") + assertArchive(t, dailiesCreated[3], time.Date(2017, 8, 13, 0, 0, 0, 0, time.UTC), DayPeriod, 1, 311, "9eaec21e28af92bc338d9b6bcd712109") assertArchive(t, dailiesCreated[4], time.Date(2017, 8, 14, 0, 0, 0, 0, time.UTC), DayPeriod, 0, 23, "f0d79988b7772c003d04a28bd7417a62") assert.Equal(t, 0, len(dailiesFailed)) assert.Equal(t, 2, len(monthliesCreated)) - assertArchive(t, monthliesCreated[0], time.Date(2017, 8, 1, 0, 0, 0, 0, time.UTC), MonthPeriod, 4, 553, "156e45e29b6587cb85ccf75e03800b00") + assertArchive(t, monthliesCreated[0], time.Date(2017, 8, 1, 0, 0, 0, 0, time.UTC), MonthPeriod, 4, 545, "d4ce6331f3c871d394ed3b916144ac85") assertArchive(t, monthliesCreated[1], time.Date(2017, 9, 1, 0, 0, 0, 0, time.UTC), MonthPeriod, 0, 23, "f0d79988b7772c003d04a28bd7417a62") assert.Equal(t, 0, len(monthliesFailed)) diff --git a/archives/messages.go b/archives/messages.go index 634b209..e987b97 100644 --- a/archives/messages.go +++ b/archives/messages.go @@ -18,7 +18,7 @@ const ( visibilityDeletedBySender = "X" ) -const lookupMsgs = ` +const sqlLookupMsgs = ` SELECT rec.visibility, row_to_json(rec) FROM ( SELECT mm.id, @@ -28,7 +28,7 @@ SELECT rec.visibility, row_to_json(rec) FROM ( row_to_json(channel) as channel, row_to_json(flow) as flow, CASE WHEN direction = 'I' THEN 'in' WHEN direction = 'O' THEN 'out' ELSE NULL END AS direction, - CASE WHEN msg_type = 'F' THEN 'flow' WHEN msg_type = 'V' THEN 'ivr' WHEN msg_type = 'I' THEN 'inbox' ELSE NULL END AS "type", + CASE WHEN msg_type = 'V' THEN 'voice' ELSE 'text' END AS "type", CASE WHEN status = 'I' THEN 'initializing' WHEN status = 'P' THEN 'queued' @@ -40,8 +40,8 @@ SELECT rec.visibility, row_to_json(rec) FROM ( WHEN status = 'F' THEN 'failed' WHEN status = 'S' THEN 'sent' WHEN status = 'R' THEN 'resent' - ELSE NULL - END as status, + ELSE NULL + END AS status, CASE WHEN visibility = 'V' THEN 'visible' WHEN visibility = 'A' THEN 'archived' WHEN visibility = 'D' THEN 'deleted' WHEN visibility = 'X' THEN 'deleted' ELSE NULL END as visibility, text, (select coalesce(jsonb_agg(attach_row), '[]'::jsonb) FROM (select attach_data.attachment[1] as content_type, attach_data.attachment[2] as url FROM (select regexp_matches(unnest(attachments), '^(.*?):(.*)$') attachment) as attach_data) as attach_row) as attachments, @@ -58,8 +58,7 @@ SELECT rec.visibility, row_to_json(rec) FROM ( LEFT JOIN LATERAL (select coalesce(jsonb_agg(label_row), '[]'::jsonb) as data from (select uuid, name from msgs_label ml INNER JOIN msgs_msg_labels mml ON ml.id = mml.label_id AND mml.msg_id = mm.id) as label_row) as labels_agg ON True WHERE mm.org_id = $1 AND mm.created_on >= $2 AND mm.created_on < $3 -ORDER BY created_on ASC, id ASC) rec; -` +ORDER BY created_on ASC, id ASC) rec;` // writeMessageRecords writes the messages in the archive's date range to the passed in writer func writeMessageRecords(ctx context.Context, db *sqlx.DB, archive *Archive, writer *bufio.Writer) (int, error) { @@ -69,7 +68,7 @@ func writeMessageRecords(ctx context.Context, db *sqlx.DB, archive *Archive, wri // first write our normal records var record, visibility string - rows, err := db.QueryxContext(ctx, lookupMsgs, archive.Org.ID, archive.StartDate, archive.endDate()) + rows, err := db.QueryxContext(ctx, sqlLookupMsgs, archive.Org.ID, archive.StartDate, archive.endDate()) if err != nil { return 0, errors.Wrapf(err, "error querying messages for org: %d", archive.Org.ID) } @@ -93,28 +92,18 @@ func writeMessageRecords(ctx context.Context, db *sqlx.DB, archive *Archive, wri return recordCount, nil } -const selectOrgMessagesInRange = ` -SELECT mm.id, mm.visibility -FROM msgs_msg mm +const sqlSelectOrgMessagesInRange = ` + SELECT mm.id, mm.visibility + FROM msgs_msg mm LEFT JOIN contacts_contact cc ON cc.id = mm.contact_id -WHERE mm.org_id = $1 AND mm.created_on >= $2 AND mm.created_on < $3 -ORDER BY mm.created_on ASC, mm.id ASC -` + WHERE mm.org_id = $1 AND mm.created_on >= $2 AND mm.created_on < $3 + ORDER BY mm.created_on ASC, mm.id ASC` -const deleteMessageLogs = ` -DELETE FROM channels_channellog -WHERE msg_id IN(?) -` +const sqlDeleteMessageLabels = ` +DELETE FROM msgs_msg_labels WHERE msg_id IN(?)` -const deleteMessageLabels = ` -DELETE FROM msgs_msg_labels -WHERE msg_id IN(?) -` - -const deleteMessages = ` -DELETE FROM msgs_msg -WHERE id IN(?) -` +const sqlDeleteMessages = ` +DELETE FROM msgs_msg WHERE id IN(?)` // DeleteArchivedMessages takes the passed in archive, verifies the S3 file is still present (and correct), then selects // all the messages in the archive date range, and if equal or fewer than the number archived, deletes them 100 at a time @@ -147,7 +136,7 @@ func DeleteArchivedMessages(ctx context.Context, config *Config, db *sqlx.DB, s3 } // ok, archive file looks good, let's build up our list of message ids, this may be big but we are int64s so shouldn't be too big - rows, err := db.QueryxContext(outer, selectOrgMessagesInRange, archive.OrgID, archive.StartDate, archive.endDate()) + rows, err := db.QueryxContext(outer, sqlSelectOrgMessagesInRange, archive.OrgID, archive.StartDate, archive.endDate()) if err != nil { return err } @@ -192,20 +181,14 @@ func DeleteArchivedMessages(ctx context.Context, config *Config, db *sqlx.DB, s3 return err } - // first delete any channel logs - err = executeInQuery(ctx, tx, deleteMessageLogs, idBatch) - if err != nil { - return errors.Wrap(err, "error removing channel logs") - } - - // then any labels - err = executeInQuery(ctx, tx, deleteMessageLabels, idBatch) + // first delete any labelings + err = executeInQuery(ctx, tx, sqlDeleteMessageLabels, idBatch) if err != nil { return errors.Wrap(err, "error removing message labels") } - // finally, delete our messages - err = executeInQuery(ctx, tx, deleteMessages, idBatch) + // then delete the messages themselves + err = executeInQuery(ctx, tx, sqlDeleteMessages, idBatch) if err != nil { return errors.Wrap(err, "error deleting messages") } @@ -227,7 +210,7 @@ func DeleteArchivedMessages(ctx context.Context, config *Config, db *sqlx.DB, s3 deletedOn := dates.Now() // all went well! mark our archive as no longer needing deletion - _, err = db.ExecContext(outer, setArchiveDeleted, archive.ID, deletedOn) + _, err = db.ExecContext(outer, sqlUpdateArchiveDeleted, archive.ID, deletedOn) if err != nil { return errors.Wrap(err, "error setting archive as deleted") } @@ -239,27 +222,18 @@ func DeleteArchivedMessages(ctx context.Context, config *Config, db *sqlx.DB, s3 return nil } -const selectOldOrgBroadcasts = ` -SELECT - id -FROM - msgs_broadcast -WHERE - org_id = $1 AND - created_on < $2 AND - schedule_id IS NULL -ORDER BY - created_on ASC, - id ASC -LIMIT 1000000; -` - -// DeleteBroadcasts deletes all broadcasts older than 90 days for the passed in org which have no active messages on them +const sqlSelectOldOrgBroadcasts = ` +SELECT id + FROM msgs_broadcast b + WHERE b.org_id = $1 AND b.created_on < $2 AND b.schedule_id IS NULL AND NOT EXISTS (SELECT 1 FROM msgs_msg WHERE broadcast_id = b.id) + LIMIT 1000000;` + +// DeleteBroadcasts deletes all broadcasts older than 90 days for the passed in org which have no associated messages func DeleteBroadcasts(ctx context.Context, now time.Time, config *Config, db *sqlx.DB, org Org) error { start := dates.Now() threshhold := now.AddDate(0, 0, -org.RetentionPeriod) - rows, err := db.QueryxContext(ctx, selectOldOrgBroadcasts, org.ID, threshhold) + rows, err := db.QueryxContext(ctx, sqlSelectOldOrgBroadcasts, org.ID, threshhold) if err != nil { return err } @@ -277,23 +251,10 @@ func DeleteBroadcasts(ctx context.Context, now time.Time, config *Config, db *sq } var broadcastID int64 - err := rows.Scan(&broadcastID) - if err != nil { + if err := rows.Scan(&broadcastID); err != nil { return errors.Wrap(err, "unable to get broadcast id") } - // make sure we have no active messages - var msgCount int64 - err = db.Get(&msgCount, `SELECT count(*) FROM msgs_msg WHERE broadcast_id = $1`, broadcastID) - if err != nil { - return errors.Wrapf(err, "unable to select number of msgs for broadcast: %d", broadcastID) - } - - if msgCount != 0 { - logrus.WithField("broadcast_id", broadcastID).WithField("org_id", org.ID).WithField("msg_count", msgCount).Warn("unable to delete broadcast, has messages still") - continue - } - // we delete broadcasts in a transaction per broadcast tx, err := db.BeginTx(ctx, nil) if err != nil { @@ -314,13 +275,6 @@ func DeleteBroadcasts(ctx context.Context, now time.Time, config *Config, db *sq return errors.Wrapf(err, "error deleting related groups for broadcast: %d", broadcastID) } - // delete URNs M2M - _, err = tx.Exec(`DELETE from msgs_broadcast_urns WHERE broadcast_id = $1`, broadcastID) - if err != nil { - tx.Rollback() - return errors.Wrapf(err, "error deleting related urns for broadcast: %d", broadcastID) - } - // delete counts associated with this broadcast _, err = tx.Exec(`DELETE from msgs_broadcastmsgcount WHERE broadcast_id = $1`, broadcastID) if err != nil { @@ -344,11 +298,7 @@ func DeleteBroadcasts(ctx context.Context, now time.Time, config *Config, db *sq } if count > 0 { - logrus.WithFields(logrus.Fields{ - "elapsed": dates.Since(start), - "count": count, - "org_id": org.ID, - }).Info("completed deleting broadcasts") + logrus.WithFields(logrus.Fields{"elapsed": dates.Since(start), "count": count, "org_id": org.ID}).Info("completed deleting broadcasts") } return nil diff --git a/archives/runs.go b/archives/runs.go index 6cab301..87eabfc 100644 --- a/archives/runs.go +++ b/archives/runs.go @@ -22,7 +22,7 @@ const ( RunStatusFailed = "F" ) -const lookupFlowRuns = ` +const sqlLookupRuns = ` SELECT rec.uuid, rec.exited_on, row_to_json(rec) FROM ( SELECT @@ -58,13 +58,12 @@ FROM ( WHERE fr.org_id = $1 AND fr.modified_on >= $2 AND fr.modified_on < $3 ORDER BY fr.modified_on ASC, id ASC -) as rec; -` +) as rec;` // writeRunRecords writes the runs in the archive's date range to the passed in writer func writeRunRecords(ctx context.Context, db *sqlx.DB, archive *Archive, writer *bufio.Writer) (int, error) { var rows *sqlx.Rows - rows, err := db.QueryxContext(ctx, lookupFlowRuns, archive.Org.ID, archive.StartDate, archive.endDate()) + rows, err := db.QueryxContext(ctx, sqlLookupRuns, archive.Org.ID, archive.StartDate, archive.endDate()) if err != nil { return 0, errors.Wrapf(err, "error querying run records for org: %d", archive.Org.ID) } @@ -96,18 +95,15 @@ func writeRunRecords(ctx context.Context, db *sqlx.DB, archive *Archive, writer return recordCount, nil } -const selectOrgRunsInRange = ` -SELECT fr.id, fr.status -FROM flows_flowrun fr +const sqlSelectOrgRunsInRange = ` + SELECT fr.id, fr.status + FROM flows_flowrun fr LEFT JOIN contacts_contact cc ON cc.id = fr.contact_id -WHERE fr.org_id = $1 AND fr.modified_on >= $2 AND fr.modified_on < $3 -ORDER BY fr.modified_on ASC, fr.id ASC -` + WHERE fr.org_id = $1 AND fr.modified_on >= $2 AND fr.modified_on < $3 + ORDER BY fr.modified_on ASC, fr.id ASC` -const deleteRuns = ` -DELETE FROM flows_flowrun -WHERE id IN(?) -` +const sqlDeleteRuns = ` +DELETE FROM flows_flowrun WHERE id IN(?)` // DeleteArchivedRuns takes the passed in archive, verifies the S3 file is still present (and correct), then selects // all the runs in the archive date range, and if equal or fewer than the number archived, deletes them 100 at a time @@ -140,7 +136,7 @@ func DeleteArchivedRuns(ctx context.Context, config *Config, db *sqlx.DB, s3Clie } // ok, archive file looks good, let's build up our list of run ids, this may be big but we are int64s so shouldn't be too big - rows, err := db.QueryxContext(outer, selectOrgRunsInRange, archive.OrgID, archive.StartDate, archive.endDate()) + rows, err := db.QueryxContext(outer, sqlSelectOrgRunsInRange, archive.OrgID, archive.StartDate, archive.endDate()) if err != nil { return err } @@ -189,7 +185,7 @@ func DeleteArchivedRuns(ctx context.Context, config *Config, db *sqlx.DB, s3Clie } // delete our runs - err = executeInQuery(ctx, tx, deleteRuns, idBatch) + err = executeInQuery(ctx, tx, sqlDeleteRuns, idBatch) if err != nil { return errors.Wrap(err, "error deleting runs") } @@ -211,7 +207,7 @@ func DeleteArchivedRuns(ctx context.Context, config *Config, db *sqlx.DB, s3Clie deletedOn := dates.Now() // all went well! mark our archive as no longer needing deletion - _, err = db.ExecContext(outer, setArchiveDeleted, archive.ID, deletedOn) + _, err = db.ExecContext(outer, sqlUpdateArchiveDeleted, archive.ID, deletedOn) if err != nil { return errors.Wrap(err, "error setting archive as deleted") } @@ -222,3 +218,92 @@ func DeleteArchivedRuns(ctx context.Context, config *Config, db *sqlx.DB, s3Clie return nil } + +const selectOldOrgFlowStarts = ` + SELECT id + FROM flows_flowstart s + WHERE s.org_id = $1 AND s.created_on < $2 AND NOT EXISTS (SELECT 1 FROM flows_flowrun WHERE start_id = s.id) + LIMIT 1000000;` + +// DeleteFlowStarts deletes all starts older than 90 days for the passed in org which have no associated runs +func DeleteFlowStarts(ctx context.Context, now time.Time, config *Config, db *sqlx.DB, org Org) error { + start := dates.Now() + threshhold := now.AddDate(0, 0, -org.RetentionPeriod) + + rows, err := db.QueryxContext(ctx, selectOldOrgFlowStarts, org.ID, threshhold) + if err != nil { + return err + } + defer rows.Close() + + count := 0 + for rows.Next() { + if count == 0 { + logrus.WithField("org_id", org.ID).Info("deleting starts") + } + + // been deleting this org more than an hour? thats enough for today, exit out + if dates.Since(start) > time.Hour { + break + } + + var startID int64 + if err := rows.Scan(&startID); err != nil { + return errors.Wrap(err, "unable to get start id") + } + + // we delete starts in a transaction per start + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return errors.Wrapf(err, "error starting transaction while deleting start: %d", startID) + } + + // delete contacts M2M + _, err = tx.Exec(`DELETE from flows_flowstart_contacts WHERE flowstart_id = $1`, startID) + if err != nil { + tx.Rollback() + return errors.Wrapf(err, "error deleting related contacts for start: %d", startID) + } + + // delete groups M2M + _, err = tx.Exec(`DELETE from flows_flowstart_groups WHERE flowstart_id = $1`, startID) + if err != nil { + tx.Rollback() + return errors.Wrapf(err, "error deleting related groups for start: %d", startID) + } + + // delete calls M2M + _, err = tx.Exec(`DELETE from flows_flowstart_calls WHERE flowstart_id = $1`, startID) + if err != nil { + tx.Rollback() + return errors.Wrapf(err, "error deleting related calls for start: %d", startID) + } + + // delete counts + _, err = tx.Exec(`DELETE from flows_flowstartcount WHERE start_id = $1`, startID) + if err != nil { + tx.Rollback() + return errors.Wrapf(err, "error deleting counts for start: %d", startID) + } + + // finally, delete our start + _, err = tx.Exec(`DELETE from flows_flowstart WHERE id = $1`, startID) + if err != nil { + tx.Rollback() + return errors.Wrapf(err, "error deleting start: %d", startID) + } + + err = tx.Commit() + if err != nil { + return errors.Wrapf(err, "error deleting start: %d", startID) + } + + count++ + } + + if count > 0 { + logrus.WithFields(logrus.Fields{"elapsed": dates.Since(start), "count": count, "org_id": org.ID}).Info("completed deleting starts") + } + + return nil +} diff --git a/archives/testdata/messages1.jsonl b/archives/testdata/messages1.jsonl index 6b53f3e..787a27b 100644 --- a/archives/testdata/messages1.jsonl +++ b/archives/testdata/messages1.jsonl @@ -1,3 +1,3 @@ -{"id":1,"broadcast":null,"contact":{"uuid":"3e814add-e614-41f7-8b5d-a07f670a698f","name":"Ajodinabiff Dane"},"urn":"tel:+12067797777","channel":{"uuid":"60f2ed5b-05f2-4156-9ff0-e44e90da1b85","name":"Channel 2"},"flow":null,"direction":"in","type":"inbox","status":"handled","visibility":"visible","text":"message 1","attachments":[],"labels":[{"name": "Label 1", "uuid": "1d9e3188-b74b-4ae0-a166-0de31aedb34a"}, {"name": "Label 2", "uuid": "c5a69101-8dc3-444f-8b0b-5ab816e46eec"}],"created_on":"2017-08-12T21:11:59.890662+00:00","sent_on":"2017-08-12T21:11:59.890662+00:00","modified_on":"2017-08-12T21:11:59.890662+00:00"} -{"id":3,"broadcast":null,"contact":{"uuid":"3e814add-e614-41f7-8b5d-a07f670a698f","name":"Ajodinabiff Dane"},"urn":"tel:+12067797777","channel":null,"flow":null,"direction":"out","type":"inbox","status":"handled","visibility":"visible","text":"message 3","attachments":[{"url": "https://foo.bar/image1.png", "content_type": "image/png"}, {"url": "https://foo.bar/image2.png", "content_type": "image/png"}],"labels":[{"name": "Label 2", "uuid": "c5a69101-8dc3-444f-8b0b-5ab816e46eec"}],"created_on":"2017-08-12T21:11:59.890662+00:00","sent_on":"2017-08-12T21:11:59.890662+00:00","modified_on":"2017-08-12T21:11:59.890662+00:00"} -{"id":9,"broadcast":null,"contact":{"uuid":"3e814add-e614-41f7-8b5d-a07f670a698f","name":"Ajodinabiff Dane"},"urn":null,"channel":null,"flow":{"uuid":"3914b88e-625b-4603-bd9f-9319dc331c6b","name":"Flow 3"},"direction":"out","type":"flow","status":"sent","visibility":"visible","text":"message 9","attachments":[],"labels":[],"created_on":"2017-08-12T21:11:59.890662+00:00","sent_on":"2017-08-12T21:11:59.890662+00:00","modified_on":"2017-08-12T21:11:59.890662+00:00"} +{"id":1,"broadcast":null,"contact":{"uuid":"3e814add-e614-41f7-8b5d-a07f670a698f","name":"Ajodinabiff Dane"},"urn":"tel:+12067797777","channel":{"uuid":"60f2ed5b-05f2-4156-9ff0-e44e90da1b85","name":"Channel 2"},"flow":null,"direction":"in","type":"text","status":"handled","visibility":"visible","text":"message 1","attachments":[],"labels":[{"name": "Label 1", "uuid": "1d9e3188-b74b-4ae0-a166-0de31aedb34a"}, {"name": "Label 2", "uuid": "c5a69101-8dc3-444f-8b0b-5ab816e46eec"}],"created_on":"2017-08-12T21:11:59.890662+00:00","sent_on":"2017-08-12T21:11:59.890662+00:00","modified_on":"2017-08-12T21:11:59.890662+00:00"} +{"id":3,"broadcast":null,"contact":{"uuid":"3e814add-e614-41f7-8b5d-a07f670a698f","name":"Ajodinabiff Dane"},"urn":"tel:+12067797777","channel":null,"flow":null,"direction":"out","type":"text","status":"handled","visibility":"visible","text":"message 3","attachments":[{"url": "https://foo.bar/image1.png", "content_type": "image/png"}, {"url": "https://foo.bar/image2.png", "content_type": "image/png"}],"labels":[{"name": "Label 2", "uuid": "c5a69101-8dc3-444f-8b0b-5ab816e46eec"}],"created_on":"2017-08-12T21:11:59.890662+00:00","sent_on":"2017-08-12T21:11:59.890662+00:00","modified_on":"2017-08-12T21:11:59.890662+00:00"} +{"id":9,"broadcast":null,"contact":{"uuid":"3e814add-e614-41f7-8b5d-a07f670a698f","name":"Ajodinabiff Dane"},"urn":null,"channel":null,"flow":{"uuid":"3914b88e-625b-4603-bd9f-9319dc331c6b","name":"Flow 3"},"direction":"out","type":"text","status":"sent","visibility":"visible","text":"message 9","attachments":[],"labels":[],"created_on":"2017-08-12T21:11:59.890662+00:00","sent_on":"2017-08-12T21:11:59.890662+00:00","modified_on":"2017-08-12T21:11:59.890662+00:00"} diff --git a/archives/testdata/messages2.jsonl b/archives/testdata/messages2.jsonl index 01f5bc6..de045a2 100644 --- a/archives/testdata/messages2.jsonl +++ b/archives/testdata/messages2.jsonl @@ -1 +1 @@ -{"id":5,"broadcast":null,"contact":{"uuid":"7051dff0-0a27-49d7-af1f-4494239139e6","name":"Joanne Stone"},"urn":null,"channel":{"uuid":"b79e0054-068f-4928-a5f4-339d10a7ad5a","name":"Channel 3"},"flow":null,"direction":"in","type":"inbox","status":"handled","visibility":"visible","text":"message 5","attachments":[],"labels":[],"created_on":"2017-08-11T19:11:59.890662+00:00","sent_on":"2017-08-11T19:11:59.890662+00:00","modified_on":"2017-08-11T19:11:59.890662+00:00"} +{"id":5,"broadcast":null,"contact":{"uuid":"7051dff0-0a27-49d7-af1f-4494239139e6","name":"Joanne Stone"},"urn":null,"channel":{"uuid":"b79e0054-068f-4928-a5f4-339d10a7ad5a","name":"Channel 3"},"flow":null,"direction":"in","type":"text","status":"handled","visibility":"visible","text":"message 5","attachments":[],"labels":[],"created_on":"2017-08-11T19:11:59.890662+00:00","sent_on":"2017-08-11T19:11:59.890662+00:00","modified_on":"2017-08-11T19:11:59.890662+00:00"} diff --git a/go.mod b/go.mod index 495c4a0..0f5909d 100644 --- a/go.mod +++ b/go.mod @@ -3,15 +3,15 @@ module github.com/nyaruka/rp-archiver go 1.19 require ( - github.com/aws/aws-sdk-go v1.44.44 + github.com/aws/aws-sdk-go v1.44.217 github.com/evalphobia/logrus_sentry v0.8.2 github.com/jmoiron/sqlx v1.3.5 - github.com/lib/pq v1.10.6 + github.com/lib/pq v1.10.7 github.com/nyaruka/ezconf v0.2.1 - github.com/nyaruka/gocommon v1.24.0 + github.com/nyaruka/gocommon v1.35.0 github.com/pkg/errors v0.9.1 - github.com/sirupsen/logrus v1.8.1 - github.com/stretchr/testify v1.8.0 + github.com/sirupsen/logrus v1.9.0 + github.com/stretchr/testify v1.8.2 ) require ( @@ -25,6 +25,6 @@ require ( github.com/naoina/toml v0.1.1 // indirect github.com/nyaruka/librato v1.0.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b // indirect + golang.org/x/sys v0.6.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index efe7470..5d77f45 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/aws/aws-sdk-go v1.44.44 h1:XLEcUxILvVBYO/frO+TTCd8NIxklX/ZOzSJSBZ+b7B8= -github.com/aws/aws-sdk-go v1.44.44/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go v1.44.217 h1:FcWC56MRl+k756aH3qeMQTylSdeJ58WN0iFz3fkyRz0= +github.com/aws/aws-sdk-go v1.44.217/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d h1:S2NE3iHSwP0XV47EEXL8mWmRdEfGscSJ+7EgePNgt0s= github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -24,8 +24,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs= -github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= +github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks= @@ -34,8 +34,8 @@ github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8= github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0= github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw= -github.com/nyaruka/gocommon v1.24.0 h1:zfluRa4h+Ms3hmed2vPov+PR/vWxtrTgBUEONRuuiIQ= -github.com/nyaruka/gocommon v1.24.0/go.mod h1:g6/d9drZXDUrtRSPe2Kf8lTUS+baHt/0G0dwHq3qeIU= +github.com/nyaruka/gocommon v1.35.0 h1:ICr1kVVBEK0chVmw2yiLwrSGSmRWWFLlX1JhoEpgdp8= +github.com/nyaruka/gocommon v1.35.0/go.mod h1:HaUQmWPrZfKS9MLnXKQj28zF4KlJrzFou+DGuqT7RbE= github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0= github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -43,28 +43,52 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220614195744-fb05da6f9022 h1:0qjDla5xICC2suMtyRH/QqX3B1btXTfNsIt/i4LFgO0= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b h1:2n253B2r0pYSmEV+UNCQoPfU/FiaizQEK5Gu4Bq4JE8= -golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/goreleaser.yml b/goreleaser.yml index 698c2cf..a5023d4 100644 --- a/goreleaser.yml +++ b/goreleaser.yml @@ -2,16 +2,18 @@ build: main: ./cmd/rp-archiver/main.go binary: rp-archiver goos: - - windows - darwin - linux goarch: - amd64 - arm64 +changelog: + filters: + exclude: + - "^Update CHANGELOG.md" + archives: - - id: main - files: + - files: - LICENSE - README.md - diff --git a/testdb.sql b/testdb.sql index 4bacc63..8504520 100644 --- a/testdb.sql +++ b/testdb.sql @@ -1,6 +1,27 @@ -CREATE EXTENSION IF NOT EXISTS HSTORE; - +DROP TABLE IF EXISTS archives_archive CASCADE; +DROP TABLE IF EXISTS channels_channellog CASCADE; +DROP TABLE IF EXISTS channels_channel CASCADE; +DROP TABLE IF EXISTS flows_flowstart_contacts CASCADE; +DROP TABLE IF EXISTS flows_flowstart_groups CASCADE; +DROP TABLE IF EXISTS flows_flowstart_calls CASCADE; +DROP TABLE IF EXISTS flows_flowstart CASCADE; +DROP TABLE IF EXISTS flows_flowrun CASCADE; +DROP TABLE IF EXISTS flows_flow CASCADE; +DROP TABLE IF EXISTS msgs_broadcast_contacts CASCADE; +DROP TABLE IF EXISTS msgs_broadcast_groups CASCADE; +DROP TABLE IF EXISTS msgs_broadcastmsgcount CASCADE; +DROP TABLE IF EXISTS msgs_broadcast CASCADE; +DROP TABLE IF EXISTS msgs_label CASCADE; +DROP TABLE IF EXISTS msgs_msg_labels CASCADE; +DROP TABLE IF EXISTS msgs_msg CASCADE; +DROP TABLE IF EXISTS ivr_call CASCADE; +DROP TABLE IF EXISTS contacts_contacturn CASCADE; +DROP TABLE IF EXISTS contacts_contactgroup_contacts CASCADE; +DROP TABLE IF EXISTS contacts_contactgroup CASCADE; +DROP TABLE IF EXISTS contacts_contact CASCADE; +DROP TABLE IF EXISTS auth_user CASCADE; DROP TABLE IF EXISTS orgs_org CASCADE; + CREATE TABLE orgs_org ( id serial primary key, name character varying(255) NOT NULL, @@ -9,37 +30,32 @@ CREATE TABLE orgs_org ( created_on timestamp with time zone NOT NULL ); -DROP TABLE IF EXISTS channels_channel CASCADE; CREATE TABLE channels_channel ( id serial primary key, - name character varying(255) NOT NULL, uuid character varying(36) NOT NULL, - org_id integer references orgs_org(id) on delete cascade + org_id integer NOT NULL REFERENCES orgs_org(id), + name character varying(255) NOT NULL ); -DROP TABLE IF EXISTS contacts_contact CASCADE; CREATE TABLE contacts_contact ( id serial primary key, + uuid character varying(36) NOT NULL, + org_id integer NOT NULL REFERENCES orgs_org(id), is_active boolean NOT NULL, created_by_id integer NOT NULL, created_on timestamp with time zone NOT NULL, modified_by_id integer NOT NULL, modified_on timestamp with time zone NOT NULL, - org_id integer NOT NULL references orgs_org(id) on delete cascade, - is_blocked boolean NOT NULL, name character varying(128), language character varying(3), - uuid character varying(36) NOT NULL, - is_stopped boolean NOT NULL, fields jsonb ); -DROP TABLE IF EXISTS contacts_contacturn CASCADE; CREATE TABLE contacts_contacturn ( id serial primary key, + org_id integer NOT NULL REFERENCES orgs_org(id), contact_id integer, scheme character varying(128) NOT NULL, - org_id integer NOT NULL, priority integer NOT NULL, path character varying(255) NOT NULL, channel_id integer, @@ -48,34 +64,31 @@ CREATE TABLE contacts_contacturn ( identity character varying(255) NOT NULL ); -DROP TABLE IF EXISTS contacts_contactgroup CASCADE; CREATE TABLE contacts_contactgroup ( id serial primary key, - uuid character varying(36) NOT NULL, + uuid uuid NOT NULL, + org_id integer NOT NULL REFERENCES orgs_org(id), name character varying(128) NOT NULL ); -DROP TABLE IF EXISTS contacts_contactgroup_contacts CASCADE; CREATE TABLE contacts_contactgroup_contacts ( id serial primary key, contactgroup_id integer NOT NULL, contact_id integer NOT NULL ); -DROP TABLE IF EXISTS flows_flow CASCADE; CREATE TABLE flows_flow ( id serial primary key, uuid character varying(36) NOT NULL, + org_id integer NOT NULL REFERENCES orgs_org(id), name character varying(128) NOT NULL ); -DROP TABLE IF EXISTS channels_channellog CASCADE; -DROP TABLE IF EXISTS msgs_msg_labels CASCADE; -DROP TABLE IF EXISTS msgs_msg CASCADE; CREATE TABLE msgs_msg ( id serial primary key, + uuid uuid NULL, + org_id integer NOT NULL REFERENCES orgs_org(id), broadcast_id integer NULL, - uuid character varying(36) NULL, text text NOT NULL, high_priority boolean NULL, created_on timestamp with time zone NOT NULL, @@ -91,55 +104,39 @@ CREATE TABLE msgs_msg ( next_attempt timestamp with time zone NOT NULL, external_id character varying(255), attachments character varying(255)[], - channel_id integer references channels_channel(id) on delete cascade, - contact_id integer NOT NULL references contacts_contact(id) on delete cascade, - contact_urn_id integer NULL references contacts_contacturn(id) on delete cascade, - org_id integer NOT NULL references orgs_org(id) on delete cascade, - flow_id integer NULL references flows_flow(id) on delete cascade, - metadata text, - topup_id integer + channel_id integer REFERENCES channels_channel(id), + contact_id integer NOT NULL REFERENCES contacts_contact(id), + contact_urn_id integer NULL REFERENCES contacts_contacturn(id), + flow_id integer NULL REFERENCES flows_flow(id), + metadata text ); -DROP TABLE IF EXISTS msgs_broadcast_recipients; -DROP TABLE IF EXISTS msgs_broadcast; CREATE TABLE msgs_broadcast ( id serial primary key, - "text" hstore NOT NULL, - purged BOOLEAN NOT NULL, + org_id integer NOT NULL REFERENCES orgs_org(id), + translations jsonb NOT NULL, created_on timestamp with time zone NOT NULL, - schedule_id int NULL, - org_id integer NOT NULL references orgs_org(id) on delete cascade + schedule_id int NULL ); -DROP TABLE IF EXISTS msgs_broadcast_contacts; CREATE TABLE msgs_broadcast_contacts ( id serial primary key, - broadcast_id integer NOT NULL, - contact_id integer NOT NULL + broadcast_id integer NOT NULL REFERENCES msgs_broadcast(id), + contact_id integer NOT NULL REFERENCES contacts_contact(id) ); -DROP TABLE IF EXISTS msgs_broadcast_groups; CREATE TABLE msgs_broadcast_groups ( id serial primary key, - broadcast_id integer NOT NULL, - group_id integer NOT NULL -); - -DROP TABLE IF EXISTS msgs_broadcast_urns; -CREATE TABLE msgs_broadcast_urns ( - id serial primary key, - broadcast_id integer NOT NULL, - contacturn_id integer NOT NULL + broadcast_id integer NOT NULL REFERENCES msgs_broadcast(id), + contactgroup_id integer NOT NULL REFERENCES contacts_contactgroup(id) ); -DROP TABLE IF EXISTS msgs_broadcastmsgcount; CREATE TABLE msgs_broadcastmsgcount ( id serial primary key, - "count" integer NOT NULL, - broadcast_id integer NOT NULL + count integer NOT NULL, + broadcast_id integer NOT NULL REFERENCES msgs_broadcast(id) ); -DROP TABLE IF EXISTS msgs_label CASCADE; CREATE TABLE msgs_label ( id serial primary key, uuid character varying(36) NULL, @@ -148,37 +145,63 @@ CREATE TABLE msgs_label ( CREATE TABLE msgs_msg_labels ( id serial primary key, - msg_id integer NOT NULL references msgs_msg(id), - label_id integer NOT NULL + msg_id integer NOT NULL REFERENCES msgs_msg(id), + label_id integer NOT NULL REFERENCES msgs_label(id) ); -DROP TABLE IF EXISTS auth_user CASCADE; CREATE TABLE auth_user ( id serial primary key, username character varying(128) NOT NULL ); -DROP TABLE IF EXISTS api_webhookevent CASCADE; -DROP TABLE IF EXISTS flows_actionlog CASCADE; -DROP TABLE IF EXISTS flows_flowrun CASCADE; +CREATE TABLE ivr_call ( + id serial primary key, + org_id integer NOT NULL REFERENCES orgs_org(id), + created_on timestamp with time zone NOT NULL +); + +CREATE TABLE flows_flowstart ( + id serial primary key, + org_id integer NOT NULL REFERENCES orgs_org(id), + created_on timestamp with time zone NOT NULL +); + +CREATE TABLE flows_flowstart_contacts ( + id serial primary key, + flowstart_id integer NOT NULL REFERENCES flows_flowstart(id), + contact_id integer NOT NULL REFERENCES contacts_contact(id) +); + +CREATE TABLE flows_flowstart_groups ( + id serial primary key, + flowstart_id integer NOT NULL REFERENCES flows_flowstart(id), + contactgroup_id integer NOT NULL REFERENCES contacts_contactgroup(id) +); + +CREATE TABLE flows_flowstart_calls ( + id serial primary key, + flowstart_id integer NOT NULL REFERENCES flows_flowstart(id), + call_id integer NOT NULL REFERENCES ivr_call(id) +); + CREATE TABLE flows_flowrun ( id serial primary key, - uuid character varying(36) NOT NULL UNIQUE, + uuid uuid NOT NULL UNIQUE, + org_id integer NOT NULL REFERENCES orgs_org(id), responded boolean NOT NULL, - contact_id integer NOT NULL references contacts_contact(id), - flow_id integer NOT NULL references flows_flow(id), - org_id integer NOT NULL references orgs_org(id), + contact_id integer NOT NULL REFERENCES contacts_contact(id), + flow_id integer NOT NULL REFERENCES flows_flow(id), + start_id integer NULL REFERENCES flows_flowstart(id), results text NOT NULL, path text NOT NULL, created_on timestamp with time zone NOT NULL, modified_on timestamp with time zone NOT NULL, exited_on timestamp with time zone NULL, - submitted_by_id integer NULL references auth_user(id), + submitted_by_id integer NULL REFERENCES auth_user(id), status varchar(1) NOT NULL, delete_from_results boolean ); -DROP TABLE IF EXISTS archives_archive CASCADE; CREATE TABLE archives_archive ( id serial primary key, archive_type varchar(16) NOT NULL, @@ -196,83 +219,78 @@ CREATE TABLE archives_archive ( rollup_id integer NULL ); -CREATE TABLE channels_channellog ( - id serial primary key, - msg_id integer NOT NULL references msgs_msg(id) -); - INSERT INTO orgs_org(id, name, is_active, is_anon, created_on) VALUES (1, 'Org 1', TRUE, FALSE, '2017-11-10 21:11:59.890662+00'), (2, 'Org 2', TRUE, FALSE, '2017-08-10 21:11:59.890662+00'), (3, 'Org 3', TRUE, TRUE, '2017-08-10 21:11:59.890662+00'), (4, 'Org 4', FALSE, TRUE, '2017-08-10 21:11:59.890662+00'); -INSERT INTO channels_channel(id, uuid, name, org_id) VALUES -(1, '8c1223c3-bd43-466b-81f1-e7266a9f4465', 'Channel 1', 1), -(2, '60f2ed5b-05f2-4156-9ff0-e44e90da1b85', 'Channel 2', 2), -(3, 'b79e0054-068f-4928-a5f4-339d10a7ad5a', 'Channel 3', 3); - -INSERT INTO archives_archive(id, archive_type, created_on, start_date, period, record_count, size, hash, url, needs_deletion, build_time, org_id) VALUES -(NEXTVAL('archives_archive_id_seq'), 'message', '2017-08-10 00:00:00.000000+00', '2017-08-10 00:00:00.000000+00', 'D', 0, 0, '', '', TRUE, 0, 3), -(NEXTVAL('archives_archive_id_seq'), 'message', '2017-09-10 00:00:00.000000+00', '2017-09-10 00:00:00.000000+00', 'D', 0, 0, '', '', TRUE, 0, 3), -(NEXTVAL('archives_archive_id_seq'), 'message', '2017-09-02 00:00:00.000000+00', '2017-09-01 00:00:00.000000+00', 'M', 0, 0, '', '', TRUE, 0, 3), -(NEXTVAL('archives_archive_id_seq'), 'message', '2017-10-08 00:00:00.000000+00', '2017-10-08 00:00:00.000000+00', 'D', 0, 0, '', '', TRUE, 0, 2); - -INSERT INTO contacts_contact(id, is_active, created_by_id, created_on, modified_by_id, modified_on, org_id, is_blocked, name, language, uuid, is_stopped) VALUES -(1, TRUE, -1, '2017-11-10 21:11:59.890662+00', -1, '2017-11-10 21:11:59.890662+00', 1, FALSE, NULL, 'eng', 'c7a2dd87-a80e-420b-8431-ca48d422e924', FALSE), -(3, TRUE, -1, '2015-03-26 10:07:14.054521+00', -1, '2015-03-26 10:07:14.054521+00', 1, FALSE, NULL, NULL, '7a6606c7-ff41-4203-aa98-454a10d37209', TRUE), -(4, TRUE, -1, '2015-03-26 13:04:58.699648+00', -1, '2015-03-26 13:04:58.699648+00', 1, TRUE, NULL, NULL, '29b45297-15ad-4061-a7d4-e0b33d121541', FALSE), -(5, TRUE, -1, '2015-03-27 07:39:28.955051+00', -1, '2015-03-27 07:39:28.955051+00', 1, FALSE, 'John Doe', NULL, '51762bba-01a2-4c4e-b5cd-b182d0405cd4', FALSE), -(6, TRUE, -1, '2015-10-30 19:42:27.001837+00', -1, '2015-10-30 19:42:27.001837+00', 2, FALSE, 'Ajodinabiff Dane', NULL, '3e814add-e614-41f7-8b5d-a07f670a698f', FALSE), -(7, TRUE, -1, '2017-11-10 21:11:59.890662+00', -1, '2017-11-10 21:11:59.890662+00', 3, FALSE, 'Joanne Stone', NULL, '7051dff0-0a27-49d7-af1f-4494239139e6', FALSE), -(8, TRUE, -1, '2015-03-27 13:39:43.995812+00', -1, '2015-03-27 13:39:43.995812+00', 2, FALSE, NULL, 'fre', 'b46f6e18-95b4-4984-9926-dded047f4eb3', FALSE), -(9, TRUE, -1, '2017-11-10 21:11:59.890662+00', -1, '2017-11-10 21:11:59.890662+00', 2, FALSE, NULL, NULL, '9195c8b7-6138-4d84-ac56-5192cc3d8ceb', FALSE), -(10, TRUE, -1, '2016-08-22 14:20:05.690311+00', -1, '2016-08-22 14:20:05.690311+00', 2, FALSE, 'John Arbies', NULL, '2b8bd28d-43e0-4c34-a4bb-0f10b11fdb8a', FALSE); - -INSERT INTO contacts_contacturn(id, contact_id, scheme, org_id, priority, path, display, identity) VALUES -(1, 1, 'tel', 1, 50, '+12067791111', NULL, 'tel:+12067791111'), -(2, 1, 'tel', 1, 50, '+12067792222', NULL, 'tel:+12067792222'), -(4, 3, 'tel', 1, 50, '+12067794444', NULL, 'tel:+12067794444'), -(5, 4, 'tel', 1, 50, '+12067795555', NULL, 'tel:+12067795555'), -(6, 5, 'tel', 1, 50, '+12060000556', NULL, 'tel:+12067796666'), -(7, 6, 'tel', 2, 50, '+12060005577', NULL, 'tel:+12067797777'), -(8, 7, 'tel', 3, 50, '+12067798888', NULL, 'tel:+12067798888'), -(9, 8, 'viber', 2, 90, 'viberpath==', NULL, 'viber:viberpath=='), -(10, 9, 'facebook', 2, 90, 1000001, 'funguy', 'facebook:1000001'), -(11, 10, 'twitterid', 2, 90, 1000001, 'fungal', 'twitterid:1000001'); - -INSERT INTO contacts_contactgroup(id, uuid, name) VALUES -(1, '4ea0f313-2f62-4e57-bdf0-232b5191dd57', 'Group 1'), -(2, '4c016340-468d-4675-a974-15cb7a45a5ab', 'Group 2'), -(3, 'e61b5bf7-8ddf-4e05-b0a8-4c46a6b68cff', 'Group 3'), -(4, '529bac39-550a-4d6f-817c-1833f3449007', 'Group 4'); +INSERT INTO channels_channel(id, uuid, org_id, name) VALUES +(1, '8c1223c3-bd43-466b-81f1-e7266a9f4465', 1, 'Channel 1'), +(2, '60f2ed5b-05f2-4156-9ff0-e44e90da1b85', 2, 'Channel 2'), +(3, 'b79e0054-068f-4928-a5f4-339d10a7ad5a', 3, 'Channel 3'); + +INSERT INTO archives_archive(id, org_id, archive_type, created_on, start_date, period, record_count, size, hash, url, needs_deletion, build_time) VALUES +(NEXTVAL('archives_archive_id_seq'), 3, 'message', '2017-08-10 00:00:00.000000+00', '2017-08-10 00:00:00.000000+00', 'D', 0, 0, '', '', TRUE, 0), +(NEXTVAL('archives_archive_id_seq'), 3, 'message', '2017-09-10 00:00:00.000000+00', '2017-09-10 00:00:00.000000+00', 'D', 0, 0, '', '', TRUE, 0), +(NEXTVAL('archives_archive_id_seq'), 3, 'message', '2017-09-02 00:00:00.000000+00', '2017-09-01 00:00:00.000000+00', 'M', 0, 0, '', '', TRUE, 0), +(NEXTVAL('archives_archive_id_seq'), 2, 'message', '2017-10-08 00:00:00.000000+00', '2017-10-08 00:00:00.000000+00', 'D', 0, 0, '', '', TRUE, 0); + +INSERT INTO contacts_contact(id, uuid, org_id, is_active, created_by_id, created_on, modified_by_id, modified_on, name, language) VALUES +(1, 'c7a2dd87-a80e-420b-8431-ca48d422e924', 1, TRUE, -1, '2017-11-10 21:11:59.890662+00', -1, '2017-11-10 21:11:59.890662+00', NULL, 'eng'), +(3, '7a6606c7-ff41-4203-aa98-454a10d37209', 1, TRUE, -1, '2015-03-26 10:07:14.054521+00', -1, '2015-03-26 10:07:14.054521+00', NULL, NULL), +(4, '29b45297-15ad-4061-a7d4-e0b33d121541', 1, TRUE, -1, '2015-03-26 13:04:58.699648+00', -1, '2015-03-26 13:04:58.699648+00', NULL, NULL), +(5, '51762bba-01a2-4c4e-b5cd-b182d0405cd4', 1, TRUE, -1, '2015-03-27 07:39:28.955051+00', -1, '2015-03-27 07:39:28.955051+00', 'John Doe', NULL), +(6, '3e814add-e614-41f7-8b5d-a07f670a698f', 2, TRUE, -1, '2015-10-30 19:42:27.001837+00', -1, '2015-10-30 19:42:27.001837+00', 'Ajodinabiff Dane', NULL), +(7, '7051dff0-0a27-49d7-af1f-4494239139e6', 3, TRUE, -1, '2017-11-10 21:11:59.890662+00', -1, '2017-11-10 21:11:59.890662+00', 'Joanne Stone', NULL), +(8, 'b46f6e18-95b4-4984-9926-dded047f4eb3', 2, TRUE, -1, '2015-03-27 13:39:43.995812+00', -1, '2015-03-27 13:39:43.995812+00', NULL, 'fre'), +(9, '9195c8b7-6138-4d84-ac56-5192cc3d8ceb', 2, TRUE, -1, '2017-11-10 21:11:59.890662+00', -1, '2017-11-10 21:11:59.890662+00', NULL, NULL), +(10, '2b8bd28d-43e0-4c34-a4bb-0f10b11fdb8a', 2, TRUE, -1, '2016-08-22 14:20:05.690311+00', -1, '2016-08-22 14:20:05.690311+00', 'John Arbies', NULL); + +INSERT INTO contacts_contacturn(id, org_id, contact_id, scheme, priority, path, display, identity) VALUES +(1, 1, 1, 'tel', 50, '+12067791111', NULL, 'tel:+12067791111'), +(2, 1, 1, 'tel', 50, '+12067792222', NULL, 'tel:+12067792222'), +(4, 1, 3, 'tel', 50, '+12067794444', NULL, 'tel:+12067794444'), +(5, 1, 4, 'tel', 50, '+12067795555', NULL, 'tel:+12067795555'), +(6, 1, 5, 'tel', 50, '+12060000556', NULL, 'tel:+12067796666'), +(7, 2, 6, 'tel', 50, '+12060005577', NULL, 'tel:+12067797777'), +(8, 3, 7, 'tel', 50, '+12067798888', NULL, 'tel:+12067798888'), +(9, 2, 8, 'viber', 90, 'viberpath==', NULL, 'viber:viberpath=='), +(10, 2, 9, 'facebook', 90, 1000001, 'funguy', 'facebook:1000001'), +(11, 2, 10, 'twitterid', 90, 1000001, 'fungal', 'twitterid:1000001'); + +INSERT INTO contacts_contactgroup(id, uuid, org_id, name) VALUES +(1, '4ea0f313-2f62-4e57-bdf0-232b5191dd57', 2, 'Group 1'), +(2, '4c016340-468d-4675-a974-15cb7a45a5ab', 2, 'Group 2'), +(3, 'e61b5bf7-8ddf-4e05-b0a8-4c46a6b68cff', 2, 'Group 3'), +(4, '529bac39-550a-4d6f-817c-1833f3449007', 2, 'Group 4'); INSERT INTO contacts_contactgroup_contacts(id, contact_id, contactgroup_id) VALUES (1, 1, 1), (3, 1, 4), (4, 3, 4); -INSERT INTO flows_flow(id, uuid, name) VALUES -(1, '6639286a-9120-45d4-aa39-03ae3942a4a6', 'Flow 1'), -(2, '629db399-a5fb-4fa0-88e6-f479957b63d2', 'Flow 2'), -(3, '3914b88e-625b-4603-bd9f-9319dc331c6b', 'Flow 3'), -(4, 'cfa2371d-2f06-481d-84b2-d974f3803bb0', 'Flow 4'); - -INSERT INTO msgs_broadcast(id, text, created_on, purged, org_id, schedule_id) VALUES -(1, 'eng=>"hello",fre=>"bonjour"'::hstore, '2017-08-12 22:11:59.890662+02:00', TRUE, 2, 1), -(2, 'base=>"hola"'::hstore, '2017-08-12 22:11:59.890662+02:00', TRUE, 2, NULL), -(3, 'base=>"not purged"'::hstore, '2017-08-12 19:11:59.890662+02:00', FALSE, 2, NULL), -(4, 'base=>"new"'::hstore, '2019-08-12 19:11:59.890662+02:00', FALSE, 2, NULL); - -INSERT INTO msgs_msg(id, broadcast_id, uuid, text, created_on, sent_on, modified_on, direction, status, visibility, msg_type, attachments, channel_id, contact_id, contact_urn_id, org_id, flow_id, msg_count, error_count, next_attempt) VALUES -(1, NULL, '2f969340-704a-4aa2-a1bd-2f832a21d257', 'message 1', '2017-08-12 21:11:59.890662+00', '2017-08-12 21:11:59.890662+00', '2017-08-12 21:11:59.890662+00', 'I', 'H', 'V', 'I', NULL, 2, 6, 7, 2, NULL, 1, 0, '2017-08-12 21:11:59.890662+00'), -(2, NULL, 'abe87ac1-015c-4803-be29-1e89509fe682', 'message 2', '2017-08-12 21:11:59.890662+00', '2017-08-12 21:11:59.890662+00', '2017-08-12 21:11:59.890662+00', 'I', 'H', 'D', 'I', NULL, 2, 6, 7, 2, NULL, 1, 0, '2017-08-12 21:11:59.890662+00'), -(3, NULL, 'a7e83a22-a6ff-4e18-82d0-19545640ccba', 'message 3', '2017-08-12 21:11:59.890662+00', '2017-08-12 21:11:59.890662+00', '2017-08-12 21:11:59.890662+00', 'O', 'H', 'V', 'I', '{"image/png:https://foo.bar/image1.png", "image/png:https://foo.bar/image2.png"}', NULL, 6, 7, 2, NULL, 1, 0, '2017-08-12 21:11:59.890662+00'), -(4, NULL, '1cad36af-5581-4c8a-81cd-83708398f61e', 'message 4', '2017-08-13 21:11:59.890662+00', '2017-08-13 21:11:59.890662+00', '2017-08-13 21:11:59.890662+00', 'I', 'H', 'V', 'I', NULL, 2, 6, 7, 2, NULL, 1, 0, '2017-08-13 21:11:59.890662+00'), -(5, NULL, 'f557972e-2eb5-42fa-9b87-902116d18787', 'message 5', '2017-08-11 21:11:59.890662+02:00', '2017-08-11 21:11:59.890662+02:00', '2017-08-11 21:11:59.890662+02:00', 'I', 'H', 'V', 'I', NULL, 3, 7, 8, 3, NULL, 1, 0, '2017-08-11 21:11:59.890662+02:00'), -(6, 2, '579d148c-0ab1-4afb-832f-afb1fe0e19b7', 'message 6', '2017-10-08 21:11:59.890662+00', '2017-10-08 21:11:59.890662+00', '2017-10-08 21:11:59.890662+00', 'I', 'H', 'V', 'I', NULL, 2, 6, 7, 2, NULL, 1, 0, '2017-10-08 21:11:59.890662+00'), -(7, NULL, '7aeca469-2593-444e-afe4-4702317534c9', 'message 7', '2018-01-02 21:11:59.890662+00', '2018-01-02 21:11:59.890662+00', '2018-01-02 21:11:59.890662+00', 'I', 'H', 'X', 'F', NULL, 2, 6, 7, 2, 2, 1, 0, '2018-01-02 21:11:59.890662+00'), -(9, NULL, 'e14ab466-0d3b-436d-a0f7-5851fd7d9b7d', 'message 9', '2017-08-12 21:11:59.890662+00', '2017-08-12 21:11:59.890662+00', '2017-08-12 21:11:59.890662+00', 'O', 'S', 'V', 'F', NULL, NULL, 6, NULL, 2, 3, 1, 0, '2017-08-12 21:11:59.890662+00'); +INSERT INTO flows_flow(id, uuid, org_id, name) VALUES +(1, '6639286a-9120-45d4-aa39-03ae3942a4a6', 2, 'Flow 1'), +(2, '629db399-a5fb-4fa0-88e6-f479957b63d2', 2, 'Flow 2'), +(3, '3914b88e-625b-4603-bd9f-9319dc331c6b', 2, 'Flow 3'), +(4, 'cfa2371d-2f06-481d-84b2-d974f3803bb0', 2, 'Flow 4'); + +INSERT INTO msgs_broadcast(id, org_id, translations, created_on, schedule_id) VALUES +(1, 2, '{"text": {"eng": "hello", "fre": "bonjour"}}', '2017-08-12 22:11:59.890662+02:00', 1), +(2, 2, '{"text": {"und": "hola"}}', '2017-08-12 22:11:59.890662+02:00', NULL), +(3, 2, '{"text": {"und": "not purged"}}', '2017-08-12 19:11:59.890662+02:00', NULL), +(4, 2, '{"text": {"und": "new"}}', '2019-08-12 19:11:59.890662+02:00', NULL); + +INSERT INTO msgs_msg(id, uuid, org_id, broadcast_id, text, created_on, sent_on, modified_on, direction, status, visibility, msg_type, attachments, channel_id, contact_id, contact_urn_id, flow_id, msg_count, error_count, next_attempt) VALUES +(1, '2f969340-704a-4aa2-a1bd-2f832a21d257', 2, NULL, 'message 1', '2017-08-12 21:11:59.890662+00', '2017-08-12 21:11:59.890662+00', '2017-08-12 21:11:59.890662+00', 'I', 'H', 'V', 'I', NULL, 2, 6, 7, NULL, 1, 0, '2017-08-12 21:11:59.890662+00'), +(2, 'abe87ac1-015c-4803-be29-1e89509fe682', 2, NULL, 'message 2', '2017-08-12 21:11:59.890662+00', '2017-08-12 21:11:59.890662+00', '2017-08-12 21:11:59.890662+00', 'I', 'H', 'D', 'I', NULL, 2, 6, 7, NULL, 1, 0, '2017-08-12 21:11:59.890662+00'), +(3, 'a7e83a22-a6ff-4e18-82d0-19545640ccba', 2, NULL, 'message 3', '2017-08-12 21:11:59.890662+00', '2017-08-12 21:11:59.890662+00', '2017-08-12 21:11:59.890662+00', 'O', 'H', 'V', 'I', '{"image/png:https://foo.bar/image1.png", "image/png:https://foo.bar/image2.png"}', NULL, 6, 7, NULL, 1, 0, '2017-08-12 21:11:59.890662+00'), +(4, '1cad36af-5581-4c8a-81cd-83708398f61e', 2, NULL, 'message 4', '2017-08-13 21:11:59.890662+00', '2017-08-13 21:11:59.890662+00', '2017-08-13 21:11:59.890662+00', 'I', 'H', 'V', 'I', NULL, 2, 6, 7, NULL, 1, 0, '2017-08-13 21:11:59.890662+00'), +(5, 'f557972e-2eb5-42fa-9b87-902116d18787', 3, NULL, 'message 5', '2017-08-11 21:11:59.890662+02:00', '2017-08-11 21:11:59.890662+02:00', '2017-08-11 21:11:59.890662+02:00', 'I', 'H', 'V', 'I', NULL, 3, 7, 8, NULL, 1, 0, '2017-08-11 21:11:59.890662+02:00'), +(6, '579d148c-0ab1-4afb-832f-afb1fe0e19b7', 2, 2, 'message 6', '2017-10-08 21:11:59.890662+00', '2017-10-08 21:11:59.890662+00', '2017-10-08 21:11:59.890662+00', 'I', 'H', 'V', 'I', NULL, 2, 6, 7, NULL, 1, 0, '2017-10-08 21:11:59.890662+00'), +(7, '7aeca469-2593-444e-afe4-4702317534c9', 2, NULL, 'message 7', '2018-01-02 21:11:59.890662+00', '2018-01-02 21:11:59.890662+00', '2018-01-02 21:11:59.890662+00', 'I', 'H', 'X', 'F', NULL, 2, 6, 7, 2, 1, 0, '2018-01-02 21:11:59.890662+00'), +(9, 'e14ab466-0d3b-436d-a0f7-5851fd7d9b7d', 2, NULL, 'message 9', '2017-08-12 21:11:59.890662+00', '2017-08-12 21:11:59.890662+00', '2017-08-12 21:11:59.890662+00', 'O', 'S', 'V', 'F', NULL, NULL, 6, NULL, 3, 1, 0, '2017-08-12 21:11:59.890662+00'); INSERT INTO msgs_label(id, uuid, name) VALUES (1, '1d9e3188-b74b-4ae0-a166-0de31aedb34a', 'Label 1'), @@ -285,34 +303,41 @@ INSERT INTO msgs_msg_labels(id, msg_id, label_id) VALUES (3, 2, 2), (4, 3, 2); -INSERT INTO channels_channellog(id, msg_id) VALUES -(1, 1), -(2, 2), -(3, 3), -(4, 4), -(5, 5), -(6, 6); - INSERT INTO auth_user(id, username) VALUES (1, 'greg@gmail.com'); -INSERT INTO flows_flowrun(id, uuid, responded, contact_id, flow_id, org_id, results, path, created_on, modified_on, exited_on, status, submitted_by_id) VALUES -(1, '4ced1260-9cfe-4b7f-81dd-b637108f15b9', TRUE, 6, 1, 2, '{}', '[]', '2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00', 'C', NULL), -(2, '7d68469c-0494-498a-bdf3-bac68321fd6d', TRUE, 6, 1, 2, +INSERT INTO ivr_call(id, org_id, created_on) VALUES +(1, 2, NOW()); + +INSERT INTO flows_flowstart(id, org_id, created_on) VALUES +(1, 2, NOW()); + +INSERT INTO flows_flowstart_contacts(flowstart_id, contact_id) VALUES +(1, 6); + +INSERT INTO flows_flowstart_groups(flowstart_id, contactgroup_id) VALUES +(1, 1); + +INSERT INTO flows_flowstart_calls(flowstart_id, call_id) VALUES +(1, 1); + +INSERT INTO flows_flowrun(id, uuid, org_id, responded, contact_id, flow_id, results, path, created_on, modified_on, exited_on, status, submitted_by_id, start_id) VALUES +(1, '4ced1260-9cfe-4b7f-81dd-b637108f15b9', 2, TRUE, 6, 1, '{}', '[]', '2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00', 'C', NULL, 1), +(2, '7d68469c-0494-498a-bdf3-bac68321fd6d', 2, TRUE, 6, 1, '{"agree": {"category": "Strongly agree", "node_uuid": "a0434c54-3e26-4eb0-bafc-46cdeaf435ac", "name": "Do you agree?", "value": "A", "created_on": "2017-05-03T12:25:21.714339+00:00", "input": "A"}}', '[{"uuid": "c3d0b417-db75-417c-8050-33776ec8f620", "node_uuid": "10896d63-8df7-4022-88dd-a9d93edf355b", "arrived_on": "2017-08-12T15:07:24.049815+02:00", "exit_uuid": "2f890507-2ad2-4bd1-92fc-0ca031155fca"}]', -'2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00', 'C', NULL), -(3, 'de782b35-a398-46ed-8550-34c66053841b', TRUE, 7, 2, 3, +'2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00', 'C', NULL, NULL), +(3, 'de782b35-a398-46ed-8550-34c66053841b', 3, TRUE, 7, 2, '{"agree": {"category": "Strongly agree", "node_uuid": "084c8cf1-715d-4d0a-b38d-a616ed74e638", "name": "Agree", "value": "A", "created_on": "2017-05-03T12:25:21.714339+00:00", "input": "A"}, "confirm_agree": {"category": "Confirmed Strongly agree", "node_uuid": "a0434c54-3e26-4eb0-bafc-46cdeaf435ab", "name": "Do you agree?", "value": "A", "created_on": "2017-05-03T12:25:21.714339+00:00", "input": "A"}}', '[{"uuid": "600ac5b4-4895-4161-ad97-6e2f1bb48bcb", "node_uuid": "accbc6e2-b0df-46cd-9a76-bff0fdf4d753", "arrived_on": "2017-08-12T15:07:24.049815+02:00", "exit_uuid": "8249e2dc-c893-4200-b6d2-398d07a459bc"}]', -'2017-08-10 21:11:59.890662+02:00','2017-08-10 21:11:59.890662+02:00','2017-08-10 21:11:59.890662+02:00', 'C', 1), -(4, '329a5d24-64fc-479c-8d24-9674c9b46530', TRUE, 7, 2, 3, +'2017-08-10 21:11:59.890662+02:00','2017-08-10 21:11:59.890662+02:00','2017-08-10 21:11:59.890662+02:00', 'C', 1, NULL), +(4, '329a5d24-64fc-479c-8d24-9674c9b46530', 3, TRUE, 7, 2, '{"agree": {"category": "Disagree", "node_uuid": "084c8cf1-715d-4d0a-b38d-a616ed74e638", "name": "Agree", "value": "B", "created_on": "2017-10-10T12:25:21.714339+00:00", "input": "B"}}', '[{"uuid": "babf4fc8-e12c-4bb9-a9dd-61178a118b5a", "node_uuid": "accbc6e2-b0df-46cd-9a76-bff0fdf4d753", "arrived_on": "2017-10-12T15:07:24.049815+02:00", "exit_uuid": "8249e2dc-c893-4200-b6d2-398d07a459bc"}]', -'2017-10-10 21:11:59.890662+02:00','2017-10-10 21:11:59.890662+02:00','2017-10-10 21:11:59.890662+02:00', 'C', NULL), -(5, 'abed67d2-06b8-4749-8bb9-ecda037b673b', TRUE, 7, 2, 3, '{}', '[]', '2017-10-10 21:11:59.890663+02:00','2017-10-10 21:11:59.890662+02:00','2017-10-10 21:11:59.890662+02:00', 'C', NULL), -(6, '6262eefe-a6e9-4201-9b76-a7f25e3b7f29', TRUE, 7, 2, 3, '{}', '[]', '2017-12-12 21:11:59.890662+02:00','2017-12-12 21:11:59.890662+02:00','2017-12-12 21:11:59.890662+02:00', 'C', NULL), -(7, '6c0d7db9-076b-4edc-ab4b-38576ae394fc', TRUE, 7, 2, 2, '{}', '[]', '2017-08-13 13:11:59.890662+02:00','2017-08-14 16:11:59.890662+02:00', NULL, 'W', NULL); +'2017-10-10 21:11:59.890662+02:00','2017-10-10 21:11:59.890662+02:00','2017-10-10 21:11:59.890662+02:00', 'C', NULL, NULL), +(5, 'abed67d2-06b8-4749-8bb9-ecda037b673b', 3, TRUE, 7, 2, '{}', '[]', '2017-10-10 21:11:59.890663+02:00','2017-10-10 21:11:59.890662+02:00','2017-10-10 21:11:59.890662+02:00', 'C', NULL, NULL), +(6, '6262eefe-a6e9-4201-9b76-a7f25e3b7f29', 3, TRUE, 7, 2, '{}', '[]', '2017-12-12 21:11:59.890662+02:00','2017-12-12 21:11:59.890662+02:00','2017-12-12 21:11:59.890662+02:00', 'C', NULL, NULL), +(7, '6c0d7db9-076b-4edc-ab4b-38576ae394fc', 2, TRUE, 7, 2, '{}', '[]', '2017-08-13 13:11:59.890662+02:00','2017-08-14 16:11:59.890662+02:00', NULL, 'W', NULL, NULL); -- update run #5 to have a path longer than 500 steps UPDATE flows_flowrun SET path = s.path FROM (