diff --git a/.circleci/config.yml b/.circleci/config.yml index ede02ac6..1d46d729 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,7 +5,18 @@ orbs: azure-cli: circleci/azure-cli@1.1.0 jobs: - + + integration-athena: + docker: + - image: cimg/python:3.9.9 + steps: + - checkout + - run: + name: "Run Tests - Athena" + command: ./run_test.sh athena + - store_artifacts: + path: ./logs + integration-redshift: docker: - image: cimg/python:3.9.9 @@ -111,6 +122,7 @@ workflows: - integration-snowflake - integration-bigquery - integration-databricks + - integration-athena #- integration-synapse #- integration-azuresql: # requires: diff --git a/README.md b/README.md index fbe4357b..2d47e8d7 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ This package provides: * Spark * Synapse * Azure SQL +* AWS Athena ![sample docs](etc/sample_docs.png) @@ -56,6 +57,7 @@ The macros assume that you: - an accessible set of files (Spark) 2. Have the appropriate permissions on to create tables using that scaffolding 3. Have already created the database/project and/or schema/dataset in which dbt will create external tables (or snowpiped tables) +4. Have set the top-level key `query-comment:` to an empty value in your `dbt_project.yml` file (Athena only) ## Spec diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index 0bd21272..1b0a394d 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -10,6 +10,15 @@ integration_tests: target: postgres outputs: + athena: + type: athena + database: "{{ env_var('ATHENA_TEST_DBNAME') }}" + region_name: "{{ env_var('AWS_REGION') }}" + s3_staging_dir: "s3://{{ env_var('ATHENA_TEST_BUCKET') }}" + work_group: "{{ env_var('ATHENA_TEST_WORKGROUP') }}" + schema: dbt_external_tables_integration_tests_athena + threads: 1 + redshift: type: redshift host: "{{ env_var('REDSHIFT_TEST_HOST') }}" diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 42cd38c0..18631991 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -24,9 +24,18 @@ dispatch: seeds: +quote_columns: false +# FIXME: query-comment must be disabled for Athena to work because /* block comments are unsupported in Athena DML +# Removing this line will result in a Runtime Error during the integration test +# `2 of 5 (2) create external table dbt.people_csv_partitioned ...`. The error is +# "FAILED: ParseException line 1:0 cannot recognize input near '/' '*' '{". +# Is there a better way around this? +query-comment: + sources: dbt_external_tables_integration_tests: plugins: + athena: + +enabled: "{{ target.type == 'athena' }}" redshift: +enabled: "{{ target.type == 'redshift' }}" snowflake: @@ -43,6 +52,8 @@ sources: tests: dbt_external_tables_integration_tests: plugins: + athena: + +enabled: "{{ target.type == 'athena' }}" redshift: +enabled: "{{ target.type == 'redshift' }}" snowflake: diff --git a/integration_tests/models/plugins/athena/athena_external.yml b/integration_tests/models/plugins/athena/athena_external.yml new file mode 100644 index 00000000..1755c096 --- /dev/null +++ b/integration_tests/models/plugins/athena/athena_external.yml @@ -0,0 +1,116 @@ +version: 2 + +sources: + - name: athena_external + schema: "{{ target.schema }}" + tables: + - name: people_csv_unpartitioned + external: &csv-people + location: "s3://dbt-external-tables-testing/csv/" + row_format: serde 'org.apache.hadoop.hive.serde2.OpenCSVSerde' + table_properties: "('skip.header.line.count'='1')" + columns: &cols-of-the-people + - name: id + data_type: int + - name: first_name + data_type: varchar(64) + - name: last_name + data_type: varchar(64) + - name: email + data_type: varchar(64) + tests: &equal-to-the-people + - dbt_utils.equality: + compare_model: ref('people') + compare_columns: + - id + - first_name + - last_name + - email + + - name: people_csv_partitioned + external: + <<: *csv-people + partitions: &parts-of-the-people + - name: section + data_type: varchar(1) + vals: ['a','b','c','d'] + path_macro: dbt_external_tables.key_value + columns: *cols-of-the-people + tests: *equal-to-the-people + + # ensure that all partitions are created + - name: people_csv_multipartitioned + external: + <<: *csv-people + location: "s3://dbt-external-tables-testing/" + partitions: + - name: file_format + data_type: varchar(4) + vals: ['csv', 'json'] + path_macro: dbt_external_tables.value_only + - name: section + data_type: varchar(1) + vals: ['a','b','c','d'] + path_macro: dbt_external_tables.key_value + - name: some_date + data_type: date + vals: + macro: dbt.dates_in_range + args: + start_date_str: '2020-01-01' + end_date_str: '2020-02-01' + in_fmt: "%Y-%m-%d" + out_fmt: "%Y-%m-%d" + path_macro: dbt_external_tables.year_month_day + - name: file_name + data_type: varchar(10) + vals: ['people', 'not_people'] + path_macro: dbt_external_tables.value_only + columns: *cols-of-the-people + + - name: people_csv_multipartitioned_hive_compatible + external: + <<: *csv-people + hive_compatible_partitions: true + location: "s3://dbt-external-tables-testing/" + partitions: + - name: file_format + data_type: varchar(4) + vals: ['csv', 'json'] + path_macro: dbt_external_tables.value_only + - name: section + data_type: varchar(1) + vals: ['a','b','c','d'] + path_macro: dbt_external_tables.key_value + - name: some_date + data_type: date + vals: + macro: dbt.dates_in_range + args: + start_date_str: '2020-01-01' + end_date_str: '2020-02-01' + in_fmt: "%Y-%m-%d" + out_fmt: "%Y-%m-%d" + path_macro: dbt_external_tables.year_month_day + - name: file_name + data_type: varchar(10) + vals: ['people', 'not_people'] + path_macro: dbt_external_tables.value_only + columns: *cols-of-the-people + + - name: people_json_unpartitioned + external: &json-people + location: "s3://dbt-external-tables-testing/json/" + row_format: "serde 'org.openx.data.jsonserde.JsonSerDe' + with serdeproperties ( + 'strip.outer.array'='false' + )" + columns: *cols-of-the-people + tests: *equal-to-the-people + + - name: people_json_partitioned + external: + <<: *json-people + partitions: *parts-of-the-people + columns: *cols-of-the-people + tests: *equal-to-the-people diff --git a/integration_tests/public_data/json/section=a/people_a.json b/integration_tests/public_data/json/section=a/people_a.json index ad4aa905..45af00b2 100644 --- a/integration_tests/public_data/json/section=a/people_a.json +++ b/integration_tests/public_data/json/section=a/people_a.json @@ -1,300 +1,50 @@ -{ - "id": 1, - "first_name": "Jack", - "last_name": "Hunter", - "email": "jhunter0@pbs.org" -} -{ - "id": 2, - "first_name": "Kathryn", - "last_name": "Walker", - "email": "kwalker1@ezinearticles.com" -} -{ - "id": 3, - "first_name": "Gerald", - "last_name": "Ryan", - "email": "gryan2@com.com" -} -{ - "id": 4, - "first_name": "Bonnie", - "last_name": "Spencer", - "email": "bspencer3@ameblo.jp" -} -{ - "id": 5, - "first_name": "Harold", - "last_name": "Taylor", - "email": "htaylor4@people.com.cn" -} -{ - "id": 6, - "first_name": "Jacqueline", - "last_name": "Griffin", - "email": "jgriffin5@t.co" -} -{ - "id": 7, - "first_name": "Wanda", - "last_name": "Arnold", - "email": "warnold6@google.nl" -} -{ - "id": 8, - "first_name": "Craig", - "last_name": "Ortiz", - "email": "cortiz7@sciencedaily.com" -} -{ - "id": 9, - "first_name": "Gary", - "last_name": "Day", - "email": "gday8@nih.gov" -} -{ - "id": 10, - "first_name": "Rose", - "last_name": "Wright", - "email": "rwright9@yahoo.co.jp" -} -{ - "id": 11, - "first_name": "Raymond", - "last_name": "Kelley", - "email": "rkelleya@fc2.com" -} -{ - "id": 12, - "first_name": "Gerald", - "last_name": "Robinson", - "email": "grobinsonb@disqus.com" -} -{ - "id": 13, - "first_name": "Mildred", - "last_name": "Martinez", - "email": "mmartinezc@samsung.com" -} -{ - "id": 14, - "first_name": "Dennis", - "last_name": "Arnold", - "email": "darnoldd@google.com" -} -{ - "id": 15, - "first_name": "Judy", - "last_name": "Gray", - "email": "jgraye@opensource.org" -} -{ - "id": 16, - "first_name": "Theresa", - "last_name": "Garza", - "email": "tgarzaf@epa.gov" -} -{ - "id": 17, - "first_name": "Gerald", - "last_name": "Robertson", - "email": "grobertsong@csmonitor.com" -} -{ - "id": 18, - "first_name": "Philip", - "last_name": "Hernandez", - "email": "phernandezh@adobe.com" -} -{ - "id": 19, - "first_name": "Julia", - "last_name": "Gonzalez", - "email": "jgonzalezi@cam.ac.uk" -} -{ - "id": 20, - "first_name": "Andrew", - "last_name": "Davis", - "email": "adavisj@patch.com" -} -{ - "id": 21, - "first_name": "Kimberly", - "last_name": "Harper", - "email": "kharperk@foxnews.com" -} -{ - "id": 22, - "first_name": "Mark", - "last_name": "Martin", - "email": "mmartinl@marketwatch.com" -} -{ - "id": 23, - "first_name": "Cynthia", - "last_name": "Ruiz", - "email": "cruizm@google.fr" -} -{ - "id": 24, - "first_name": "Samuel", - "last_name": "Carroll", - "email": "scarrolln@youtu.be" -} -{ - "id": 25, - "first_name": "Jennifer", - "last_name": "Larson", - "email": "jlarsono@vinaora.com" -} -{ - "id": 26, - "first_name": "Ashley", - "last_name": "Perry", - "email": "aperryp@rakuten.co.jp" -} -{ - "id": 27, - "first_name": "Howard", - "last_name": "Rodriguez", - "email": "hrodriguezq@shutterfly.com" -} -{ - "id": 28, - "first_name": "Amy", - "last_name": "Brooks", - "email": "abrooksr@theatlantic.com" -} -{ - "id": 29, - "first_name": "Louise", - "last_name": "Warren", - "email": "lwarrens@adobe.com" -} -{ - "id": 30, - "first_name": "Tina", - "last_name": "Watson", - "email": "twatsont@myspace.com" -} -{ - "id": 31, - "first_name": "Janice", - "last_name": "Kelley", - "email": "jkelleyu@creativecommons.org" -} -{ - "id": 32, - "first_name": "Terry", - "last_name": "Mccoy", - "email": "tmccoyv@bravesites.com" -} -{ - "id": 33, - "first_name": "Jeffrey", - "last_name": "Morgan", - "email": "jmorganw@surveymonkey.com" -} -{ - "id": 34, - "first_name": "Louis", - "last_name": "Harvey", - "email": "lharveyx@sina.com.cn" -} -{ - "id": 35, - "first_name": "Philip", - "last_name": "Miller", - "email": "pmillery@samsung.com" -} -{ - "id": 36, - "first_name": "Willie", - "last_name": "Marshall", - "email": "wmarshallz@ow.ly" -} -{ - "id": 37, - "first_name": "Patrick", - "last_name": "Lopez", - "email": "plopez10@redcross.org" -} -{ - "id": 38, - "first_name": "Adam", - "last_name": "Jenkins", - "email": "ajenkins11@harvard.edu" -} -{ - "id": 39, - "first_name": "Benjamin", - "last_name": "Cruz", - "email": "bcruz12@linkedin.com" -} -{ - "id": 40, - "first_name": "Ruby", - "last_name": "Hawkins", - "email": "rhawkins13@gmpg.org" -} -{ - "id": 41, - "first_name": "Carlos", - "last_name": "Barnes", - "email": "cbarnes14@a8.net" -} -{ - "id": 42, - "first_name": "Ruby", - "last_name": "Griffin", - "email": "rgriffin15@bravesites.com" -} -{ - "id": 43, - "first_name": "Sean", - "last_name": "Mason", - "email": "smason16@icq.com" -} -{ - "id": 44, - "first_name": "Anthony", - "last_name": "Payne", - "email": "apayne17@utexas.edu" -} -{ - "id": 45, - "first_name": "Steve", - "last_name": "Cruz", - "email": "scruz18@pcworld.com" -} -{ - "id": 46, - "first_name": "Anthony", - "last_name": "Garcia", - "email": "agarcia19@flavors.me" -} -{ - "id": 47, - "first_name": "Doris", - "last_name": "Lopez", - "email": "dlopez1a@sphinn.com" -} -{ - "id": 48, - "first_name": "Susan", - "last_name": "Nichols", - "email": "snichols1b@freewebs.com" -} -{ - "id": 49, - "first_name": "Wanda", - "last_name": "Ferguson", - "email": "wferguson1c@yahoo.co.jp" -} -{ - "id": 50, - "first_name": "Andrea", - "last_name": "Pierce", - "email": "apierce1d@google.co.uk" -} \ No newline at end of file +{"id":1,"first_name":"Jack","last_name":"Hunter","email":"jhunter0@pbs.org"} +{"id":2,"first_name":"Kathryn","last_name":"Walker","email":"kwalker1@ezinearticles.com"} +{"id":3,"first_name":"Gerald","last_name":"Ryan","email":"gryan2@com.com"} +{"id":4,"first_name":"Bonnie","last_name":"Spencer","email":"bspencer3@ameblo.jp"} +{"id":5,"first_name":"Harold","last_name":"Taylor","email":"htaylor4@people.com.cn"} +{"id":6,"first_name":"Jacqueline","last_name":"Griffin","email":"jgriffin5@t.co"} +{"id":7,"first_name":"Wanda","last_name":"Arnold","email":"warnold6@google.nl"} +{"id":8,"first_name":"Craig","last_name":"Ortiz","email":"cortiz7@sciencedaily.com"} +{"id":9,"first_name":"Gary","last_name":"Day","email":"gday8@nih.gov"} +{"id":10,"first_name":"Rose","last_name":"Wright","email":"rwright9@yahoo.co.jp"} +{"id":11,"first_name":"Raymond","last_name":"Kelley","email":"rkelleya@fc2.com"} +{"id":12,"first_name":"Gerald","last_name":"Robinson","email":"grobinsonb@disqus.com"} +{"id":13,"first_name":"Mildred","last_name":"Martinez","email":"mmartinezc@samsung.com"} +{"id":14,"first_name":"Dennis","last_name":"Arnold","email":"darnoldd@google.com"} +{"id":15,"first_name":"Judy","last_name":"Gray","email":"jgraye@opensource.org"} +{"id":16,"first_name":"Theresa","last_name":"Garza","email":"tgarzaf@epa.gov"} +{"id":17,"first_name":"Gerald","last_name":"Robertson","email":"grobertsong@csmonitor.com"} +{"id":18,"first_name":"Philip","last_name":"Hernandez","email":"phernandezh@adobe.com"} +{"id":19,"first_name":"Julia","last_name":"Gonzalez","email":"jgonzalezi@cam.ac.uk"} +{"id":20,"first_name":"Andrew","last_name":"Davis","email":"adavisj@patch.com"} +{"id":21,"first_name":"Kimberly","last_name":"Harper","email":"kharperk@foxnews.com"} +{"id":22,"first_name":"Mark","last_name":"Martin","email":"mmartinl@marketwatch.com"} +{"id":23,"first_name":"Cynthia","last_name":"Ruiz","email":"cruizm@google.fr"} +{"id":24,"first_name":"Samuel","last_name":"Carroll","email":"scarrolln@youtu.be"} +{"id":25,"first_name":"Jennifer","last_name":"Larson","email":"jlarsono@vinaora.com"} +{"id":26,"first_name":"Ashley","last_name":"Perry","email":"aperryp@rakuten.co.jp"} +{"id":27,"first_name":"Howard","last_name":"Rodriguez","email":"hrodriguezq@shutterfly.com"} +{"id":28,"first_name":"Amy","last_name":"Brooks","email":"abrooksr@theatlantic.com"} +{"id":29,"first_name":"Louise","last_name":"Warren","email":"lwarrens@adobe.com"} +{"id":30,"first_name":"Tina","last_name":"Watson","email":"twatsont@myspace.com"} +{"id":31,"first_name":"Janice","last_name":"Kelley","email":"jkelleyu@creativecommons.org"} +{"id":32,"first_name":"Terry","last_name":"Mccoy","email":"tmccoyv@bravesites.com"} +{"id":33,"first_name":"Jeffrey","last_name":"Morgan","email":"jmorganw@surveymonkey.com"} +{"id":34,"first_name":"Louis","last_name":"Harvey","email":"lharveyx@sina.com.cn"} +{"id":35,"first_name":"Philip","last_name":"Miller","email":"pmillery@samsung.com"} +{"id":36,"first_name":"Willie","last_name":"Marshall","email":"wmarshallz@ow.ly"} +{"id":37,"first_name":"Patrick","last_name":"Lopez","email":"plopez10@redcross.org"} +{"id":38,"first_name":"Adam","last_name":"Jenkins","email":"ajenkins11@harvard.edu"} +{"id":39,"first_name":"Benjamin","last_name":"Cruz","email":"bcruz12@linkedin.com"} +{"id":40,"first_name":"Ruby","last_name":"Hawkins","email":"rhawkins13@gmpg.org"} +{"id":41,"first_name":"Carlos","last_name":"Barnes","email":"cbarnes14@a8.net"} +{"id":42,"first_name":"Ruby","last_name":"Griffin","email":"rgriffin15@bravesites.com"} +{"id":43,"first_name":"Sean","last_name":"Mason","email":"smason16@icq.com"} +{"id":44,"first_name":"Anthony","last_name":"Payne","email":"apayne17@utexas.edu"} +{"id":45,"first_name":"Steve","last_name":"Cruz","email":"scruz18@pcworld.com"} +{"id":46,"first_name":"Anthony","last_name":"Garcia","email":"agarcia19@flavors.me"} +{"id":47,"first_name":"Doris","last_name":"Lopez","email":"dlopez1a@sphinn.com"} +{"id":48,"first_name":"Susan","last_name":"Nichols","email":"snichols1b@freewebs.com"} +{"id":49,"first_name":"Wanda","last_name":"Ferguson","email":"wferguson1c@yahoo.co.jp"} +{"id":50,"first_name":"Andrea","last_name":"Pierce","email":"apierce1d@google.co.uk"} \ No newline at end of file diff --git a/integration_tests/public_data/json/section=b/people_b.json b/integration_tests/public_data/json/section=b/people_b.json index 9fa6ed8f..4bbb927a 100644 --- a/integration_tests/public_data/json/section=b/people_b.json +++ b/integration_tests/public_data/json/section=b/people_b.json @@ -1,300 +1,50 @@ -{ - "id": 51, - "first_name": "Lawrence", - "last_name": "Phillips", - "email": "lphillips1e@jugem.jp" -} -{ - "id": 52, - "first_name": "Judy", - "last_name": "Gilbert", - "email": "jgilbert1f@multiply.com" -} -{ - "id": 53, - "first_name": "Eric", - "last_name": "Williams", - "email": "ewilliams1g@joomla.org" -} -{ - "id": 54, - "first_name": "Ralph", - "last_name": "Romero", - "email": "rromero1h@sogou.com" -} -{ - "id": 55, - "first_name": "Jean", - "last_name": "Wilson", - "email": "jwilson1i@ocn.ne.jp" -} -{ - "id": 56, - "first_name": "Lori", - "last_name": "Reynolds", - "email": "lreynolds1j@illinois.edu" -} -{ - "id": 57, - "first_name": "Donald", - "last_name": "Moreno", - "email": "dmoreno1k@bbc.co.uk" -} -{ - "id": 58, - "first_name": "Steven", - "last_name": "Berry", - "email": "sberry1l@eepurl.com" -} -{ - "id": 59, - "first_name": "Theresa", - "last_name": "Shaw", - "email": "tshaw1m@people.com.cn" -} -{ - "id": 60, - "first_name": "John", - "last_name": "Stephens", - "email": "jstephens1n@nationalgeographic.com" -} -{ - "id": 61, - "first_name": "Richard", - "last_name": "Jacobs", - "email": "rjacobs1o@state.tx.us" -} -{ - "id": 62, - "first_name": "Andrew", - "last_name": "Lawson", - "email": "alawson1p@over-blog.com" -} -{ - "id": 63, - "first_name": "Peter", - "last_name": "Morgan", - "email": "pmorgan1q@rambler.ru" -} -{ - "id": 64, - "first_name": "Nicole", - "last_name": "Garrett", - "email": "ngarrett1r@zimbio.com" -} -{ - "id": 65, - "first_name": "Joshua", - "last_name": "Kim", - "email": "jkim1s@edublogs.org" -} -{ - "id": 66, - "first_name": "Ralph", - "last_name": "Roberts", - "email": "rroberts1t@people.com.cn" -} -{ - "id": 67, - "first_name": "George", - "last_name": "Montgomery", - "email": "gmontgomery1u@smugmug.com" -} -{ - "id": 68, - "first_name": "Gerald", - "last_name": "Alvarez", - "email": "galvarez1v@flavors.me" -} -{ - "id": 69, - "first_name": "Donald", - "last_name": "Olson", - "email": "dolson1w@whitehouse.gov" -} -{ - "id": 70, - "first_name": "Carlos", - "last_name": "Morgan", - "email": "cmorgan1x@pbs.org" -} -{ - "id": 71, - "first_name": "Aaron", - "last_name": "Stanley", - "email": "astanley1y@webnode.com" -} -{ - "id": 72, - "first_name": "Virginia", - "last_name": "Long", - "email": "vlong1z@spiegel.de" -} -{ - "id": 73, - "first_name": "Robert", - "last_name": "Berry", - "email": "rberry20@tripadvisor.com" -} -{ - "id": 74, - "first_name": "Antonio", - "last_name": "Brooks", - "email": "abrooks21@unesco.org" -} -{ - "id": 75, - "first_name": "Ruby", - "last_name": "Garcia", - "email": "rgarcia22@ovh.net" -} -{ - "id": 76, - "first_name": "Jack", - "last_name": "Hanson", - "email": "jhanson23@blogtalkradio.com" -} -{ - "id": 77, - "first_name": "Kathryn", - "last_name": "Nelson", - "email": "knelson24@walmart.com" -} -{ - "id": 78, - "first_name": "Jason", - "last_name": "Reed", - "email": "jreed25@printfriendly.com" -} -{ - "id": 79, - "first_name": "George", - "last_name": "Coleman", - "email": "gcoleman26@people.com.cn" -} -{ - "id": 80, - "first_name": "Rose", - "last_name": "King", - "email": "rking27@ucoz.com" -} -{ - "id": 81, - "first_name": "Johnny", - "last_name": "Holmes", - "email": "jholmes28@boston.com" -} -{ - "id": 82, - "first_name": "Katherine", - "last_name": "Gilbert", - "email": "kgilbert29@altervista.org" -} -{ - "id": 83, - "first_name": "Joshua", - "last_name": "Thomas", - "email": "jthomas2a@ustream.tv" -} -{ - "id": 84, - "first_name": "Julie", - "last_name": "Perry", - "email": "jperry2b@opensource.org" -} -{ - "id": 85, - "first_name": "Richard", - "last_name": "Perry", - "email": "rperry2c@oracle.com" -} -{ - "id": 86, - "first_name": "Kenneth", - "last_name": "Ruiz", - "email": "kruiz2d@wikimedia.org" -} -{ - "id": 87, - "first_name": "Jose", - "last_name": "Morgan", - "email": "jmorgan2e@webnode.com" -} -{ - "id": 88, - "first_name": "Donald", - "last_name": "Campbell", - "email": "dcampbell2f@goo.ne.jp" -} -{ - "id": 89, - "first_name": "Debra", - "last_name": "Collins", - "email": "dcollins2g@uol.com.br" -} -{ - "id": 90, - "first_name": "Jesse", - "last_name": "Johnson", - "email": "jjohnson2h@stumbleupon.com" -} -{ - "id": 91, - "first_name": "Elizabeth", - "last_name": "Stone", - "email": "estone2i@histats.com" -} -{ - "id": 92, - "first_name": "Angela", - "last_name": "Rogers", - "email": "arogers2j@goodreads.com" -} -{ - "id": 93, - "first_name": "Emily", - "last_name": "Dixon", - "email": "edixon2k@mlb.com" -} -{ - "id": 94, - "first_name": "Albert", - "last_name": "Scott", - "email": "ascott2l@tinypic.com" -} -{ - "id": 95, - "first_name": "Barbara", - "last_name": "Peterson", - "email": "bpeterson2m@ow.ly" -} -{ - "id": 96, - "first_name": "Adam", - "last_name": "Greene", - "email": "agreene2n@fastcompany.com" -} -{ - "id": 97, - "first_name": "Earl", - "last_name": "Sanders", - "email": "esanders2o@hc360.com" -} -{ - "id": 98, - "first_name": "Angela", - "last_name": "Brooks", - "email": "abrooks2p@mtv.com" -} -{ - "id": 99, - "first_name": "Harold", - "last_name": "Foster", - "email": "hfoster2q@privacy.gov.au" -} -{ - "id": 100, - "first_name": "Carl", - "last_name": "Meyer", - "email": "cmeyer2r@disqus.com" -} \ No newline at end of file +{"id":51,"first_name":"Lawrence","last_name":"Phillips","email":"lphillips1e@jugem.jp"} +{"id":52,"first_name":"Judy","last_name":"Gilbert","email":"jgilbert1f@multiply.com"} +{"id":53,"first_name":"Eric","last_name":"Williams","email":"ewilliams1g@joomla.org"} +{"id":54,"first_name":"Ralph","last_name":"Romero","email":"rromero1h@sogou.com"} +{"id":55,"first_name":"Jean","last_name":"Wilson","email":"jwilson1i@ocn.ne.jp"} +{"id":56,"first_name":"Lori","last_name":"Reynolds","email":"lreynolds1j@illinois.edu"} +{"id":57,"first_name":"Donald","last_name":"Moreno","email":"dmoreno1k@bbc.co.uk"} +{"id":58,"first_name":"Steven","last_name":"Berry","email":"sberry1l@eepurl.com"} +{"id":59,"first_name":"Theresa","last_name":"Shaw","email":"tshaw1m@people.com.cn"} +{"id":60,"first_name":"John","last_name":"Stephens","email":"jstephens1n@nationalgeographic.com"} +{"id":61,"first_name":"Richard","last_name":"Jacobs","email":"rjacobs1o@state.tx.us"} +{"id":62,"first_name":"Andrew","last_name":"Lawson","email":"alawson1p@over-blog.com"} +{"id":63,"first_name":"Peter","last_name":"Morgan","email":"pmorgan1q@rambler.ru"} +{"id":64,"first_name":"Nicole","last_name":"Garrett","email":"ngarrett1r@zimbio.com"} +{"id":65,"first_name":"Joshua","last_name":"Kim","email":"jkim1s@edublogs.org"} +{"id":66,"first_name":"Ralph","last_name":"Roberts","email":"rroberts1t@people.com.cn"} +{"id":67,"first_name":"George","last_name":"Montgomery","email":"gmontgomery1u@smugmug.com"} +{"id":68,"first_name":"Gerald","last_name":"Alvarez","email":"galvarez1v@flavors.me"} +{"id":69,"first_name":"Donald","last_name":"Olson","email":"dolson1w@whitehouse.gov"} +{"id":70,"first_name":"Carlos","last_name":"Morgan","email":"cmorgan1x@pbs.org"} +{"id":71,"first_name":"Aaron","last_name":"Stanley","email":"astanley1y@webnode.com"} +{"id":72,"first_name":"Virginia","last_name":"Long","email":"vlong1z@spiegel.de"} +{"id":73,"first_name":"Robert","last_name":"Berry","email":"rberry20@tripadvisor.com"} +{"id":74,"first_name":"Antonio","last_name":"Brooks","email":"abrooks21@unesco.org"} +{"id":75,"first_name":"Ruby","last_name":"Garcia","email":"rgarcia22@ovh.net"} +{"id":76,"first_name":"Jack","last_name":"Hanson","email":"jhanson23@blogtalkradio.com"} +{"id":77,"first_name":"Kathryn","last_name":"Nelson","email":"knelson24@walmart.com"} +{"id":78,"first_name":"Jason","last_name":"Reed","email":"jreed25@printfriendly.com"} +{"id":79,"first_name":"George","last_name":"Coleman","email":"gcoleman26@people.com.cn"} +{"id":80,"first_name":"Rose","last_name":"King","email":"rking27@ucoz.com"} +{"id":81,"first_name":"Johnny","last_name":"Holmes","email":"jholmes28@boston.com"} +{"id":82,"first_name":"Katherine","last_name":"Gilbert","email":"kgilbert29@altervista.org"} +{"id":83,"first_name":"Joshua","last_name":"Thomas","email":"jthomas2a@ustream.tv"} +{"id":84,"first_name":"Julie","last_name":"Perry","email":"jperry2b@opensource.org"} +{"id":85,"first_name":"Richard","last_name":"Perry","email":"rperry2c@oracle.com"} +{"id":86,"first_name":"Kenneth","last_name":"Ruiz","email":"kruiz2d@wikimedia.org"} +{"id":87,"first_name":"Jose","last_name":"Morgan","email":"jmorgan2e@webnode.com"} +{"id":88,"first_name":"Donald","last_name":"Campbell","email":"dcampbell2f@goo.ne.jp"} +{"id":89,"first_name":"Debra","last_name":"Collins","email":"dcollins2g@uol.com.br"} +{"id":90,"first_name":"Jesse","last_name":"Johnson","email":"jjohnson2h@stumbleupon.com"} +{"id":91,"first_name":"Elizabeth","last_name":"Stone","email":"estone2i@histats.com"} +{"id":92,"first_name":"Angela","last_name":"Rogers","email":"arogers2j@goodreads.com"} +{"id":93,"first_name":"Emily","last_name":"Dixon","email":"edixon2k@mlb.com"} +{"id":94,"first_name":"Albert","last_name":"Scott","email":"ascott2l@tinypic.com"} +{"id":95,"first_name":"Barbara","last_name":"Peterson","email":"bpeterson2m@ow.ly"} +{"id":96,"first_name":"Adam","last_name":"Greene","email":"agreene2n@fastcompany.com"} +{"id":97,"first_name":"Earl","last_name":"Sanders","email":"esanders2o@hc360.com"} +{"id":98,"first_name":"Angela","last_name":"Brooks","email":"abrooks2p@mtv.com"} +{"id":99,"first_name":"Harold","last_name":"Foster","email":"hfoster2q@privacy.gov.au"} +{"id":100,"first_name":"Carl","last_name":"Meyer","email":"cmeyer2r@disqus.com"} \ No newline at end of file diff --git a/integration_tests/public_data/json/section=c/people_c.json b/integration_tests/public_data/json/section=c/people_c.json index 46bc4004..8bd9d29e 100644 --- a/integration_tests/public_data/json/section=c/people_c.json +++ b/integration_tests/public_data/json/section=c/people_c.json @@ -1,300 +1,50 @@ -{ - "id": 101, - "first_name": "Michael", - "last_name": "Perez", - "email": "mperez0@chronoengine.com" -} -{ - "id": 102, - "first_name": "Shawn", - "last_name": "Mccoy", - "email": "smccoy1@reddit.com" -} -{ - "id": 103, - "first_name": "Kathleen", - "last_name": "Payne", - "email": "kpayne2@cargocollective.com" -} -{ - "id": 104, - "first_name": "Jimmy", - "last_name": "Cooper", - "email": "jcooper3@cargocollective.com" -} -{ - "id": 105, - "first_name": "Katherine", - "last_name": "Rice", - "email": "krice4@typepad.com" -} -{ - "id": 106, - "first_name": "Sarah", - "last_name": "Ryan", - "email": "sryan5@gnu.org" -} -{ - "id": 107, - "first_name": "Martin", - "last_name": "Mcdonald", - "email": "mmcdonald6@opera.com" -} -{ - "id": 108, - "first_name": "Frank", - "last_name": "Robinson", - "email": "frobinson7@wunderground.com" -} -{ - "id": 109, - "first_name": "Jennifer", - "last_name": "Franklin", - "email": "jfranklin8@mail.ru" -} -{ - "id": 110, - "first_name": "Henry", - "last_name": "Welch", - "email": "hwelch9@list-manage.com" -} -{ - "id": 111, - "first_name": "Fred", - "last_name": "Snyder", - "email": "fsnydera@reddit.com" -} -{ - "id": 112, - "first_name": "Amy", - "last_name": "Dunn", - "email": "adunnb@nba.com" -} -{ - "id": 113, - "first_name": "Kathleen", - "last_name": "Meyer", - "email": "kmeyerc@cdc.gov" -} -{ - "id": 114, - "first_name": "Steve", - "last_name": "Ferguson", - "email": "sfergusond@reverbnation.com" -} -{ - "id": 115, - "first_name": "Teresa", - "last_name": "Hill", - "email": "thille@dion.ne.jp" -} -{ - "id": 116, - "first_name": "Amanda", - "last_name": "Harper", - "email": "aharperf@mail.ru" -} -{ - "id": 117, - "first_name": "Kimberly", - "last_name": "Ray", - "email": "krayg@xing.com" -} -{ - "id": 118, - "first_name": "Johnny", - "last_name": "Knight", - "email": "jknighth@jalbum.net" -} -{ - "id": 119, - "first_name": "Virginia", - "last_name": "Freeman", - "email": "vfreemani@tiny.cc" -} -{ - "id": 120, - "first_name": "Anna", - "last_name": "Austin", - "email": "aaustinj@diigo.com" -} -{ - "id": 121, - "first_name": "Willie", - "last_name": "Hill", - "email": "whillk@mail.ru" -} -{ - "id": 122, - "first_name": "Sean", - "last_name": "Harris", - "email": "sharrisl@zdnet.com" -} -{ - "id": 123, - "first_name": "Mildred", - "last_name": "Adams", - "email": "madamsm@usatoday.com" -} -{ - "id": 124, - "first_name": "David", - "last_name": "Graham", - "email": "dgrahamn@zimbio.com" -} -{ - "id": 125, - "first_name": "Victor", - "last_name": "Hunter", - "email": "vhuntero@ehow.com" -} -{ - "id": 126, - "first_name": "Aaron", - "last_name": "Ruiz", - "email": "aruizp@weebly.com" -} -{ - "id": 127, - "first_name": "Benjamin", - "last_name": "Brooks", - "email": "bbrooksq@jalbum.net" -} -{ - "id": 128, - "first_name": "Lisa", - "last_name": "Wilson", - "email": "lwilsonr@japanpost.jp" -} -{ - "id": 129, - "first_name": "Benjamin", - "last_name": "King", - "email": "bkings@comsenz.com" -} -{ - "id": 130, - "first_name": "Christina", - "last_name": "Williamson", - "email": "cwilliamsont@boston.com" -} -{ - "id": 131, - "first_name": "Jane", - "last_name": "Gonzalez", - "email": "jgonzalezu@networksolutions.com" -} -{ - "id": 132, - "first_name": "Thomas", - "last_name": "Owens", - "email": "towensv@psu.edu" -} -{ - "id": 133, - "first_name": "Katherine", - "last_name": "Moore", - "email": "kmoorew@naver.com" -} -{ - "id": 134, - "first_name": "Jennifer", - "last_name": "Stewart", - "email": "jstewartx@yahoo.com" -} -{ - "id": 135, - "first_name": "Sara", - "last_name": "Tucker", - "email": "stuckery@topsy.com" -} -{ - "id": 136, - "first_name": "Harold", - "last_name": "Ortiz", - "email": "hortizz@vkontakte.ru" -} -{ - "id": 137, - "first_name": "Shirley", - "last_name": "James", - "email": "sjames10@yelp.com" -} -{ - "id": 138, - "first_name": "Dennis", - "last_name": "Johnson", - "email": "djohnson11@slate.com" -} -{ - "id": 139, - "first_name": "Louise", - "last_name": "Weaver", - "email": "lweaver12@china.com.cn" -} -{ - "id": 140, - "first_name": "Maria", - "last_name": "Armstrong", - "email": "marmstrong13@prweb.com" -} -{ - "id": 141, - "first_name": "Gloria", - "last_name": "Cruz", - "email": "gcruz14@odnoklassniki.ru" -} -{ - "id": 142, - "first_name": "Diana", - "last_name": "Spencer", - "email": "dspencer15@ifeng.com" -} -{ - "id": 143, - "first_name": "Kelly", - "last_name": "Nguyen", - "email": "knguyen16@altervista.org" -} -{ - "id": 144, - "first_name": "Jane", - "last_name": "Rodriguez", - "email": "jrodriguez17@biblegateway.com" -} -{ - "id": 145, - "first_name": "Scott", - "last_name": "Brown", - "email": "sbrown18@geocities.jp" -} -{ - "id": 146, - "first_name": "Norma", - "last_name": "Cruz", - "email": "ncruz19@si.edu" -} -{ - "id": 147, - "first_name": "Marie", - "last_name": "Peters", - "email": "mpeters1a@mlb.com" -} -{ - "id": 148, - "first_name": "Lillian", - "last_name": "Carr", - "email": "lcarr1b@typepad.com" -} -{ - "id": 149, - "first_name": "Judy", - "last_name": "Nichols", - "email": "jnichols1c@t-online.de" -} -{ - "id": 150, - "first_name": "Billy", - "last_name": "Long", - "email": "blong1d@yahoo.com" -} \ No newline at end of file +{"id":101,"first_name":"Michael","last_name":"Perez","email":"mperez0@chronoengine.com"} +{"id":102,"first_name":"Shawn","last_name":"Mccoy","email":"smccoy1@reddit.com"} +{"id":103,"first_name":"Kathleen","last_name":"Payne","email":"kpayne2@cargocollective.com"} +{"id":104,"first_name":"Jimmy","last_name":"Cooper","email":"jcooper3@cargocollective.com"} +{"id":105,"first_name":"Katherine","last_name":"Rice","email":"krice4@typepad.com"} +{"id":106,"first_name":"Sarah","last_name":"Ryan","email":"sryan5@gnu.org"} +{"id":107,"first_name":"Martin","last_name":"Mcdonald","email":"mmcdonald6@opera.com"} +{"id":108,"first_name":"Frank","last_name":"Robinson","email":"frobinson7@wunderground.com"} +{"id":109,"first_name":"Jennifer","last_name":"Franklin","email":"jfranklin8@mail.ru"} +{"id":110,"first_name":"Henry","last_name":"Welch","email":"hwelch9@list-manage.com"} +{"id":111,"first_name":"Fred","last_name":"Snyder","email":"fsnydera@reddit.com"} +{"id":112,"first_name":"Amy","last_name":"Dunn","email":"adunnb@nba.com"} +{"id":113,"first_name":"Kathleen","last_name":"Meyer","email":"kmeyerc@cdc.gov"} +{"id":114,"first_name":"Steve","last_name":"Ferguson","email":"sfergusond@reverbnation.com"} +{"id":115,"first_name":"Teresa","last_name":"Hill","email":"thille@dion.ne.jp"} +{"id":116,"first_name":"Amanda","last_name":"Harper","email":"aharperf@mail.ru"} +{"id":117,"first_name":"Kimberly","last_name":"Ray","email":"krayg@xing.com"} +{"id":118,"first_name":"Johnny","last_name":"Knight","email":"jknighth@jalbum.net"} +{"id":119,"first_name":"Virginia","last_name":"Freeman","email":"vfreemani@tiny.cc"} +{"id":120,"first_name":"Anna","last_name":"Austin","email":"aaustinj@diigo.com"} +{"id":121,"first_name":"Willie","last_name":"Hill","email":"whillk@mail.ru"} +{"id":122,"first_name":"Sean","last_name":"Harris","email":"sharrisl@zdnet.com"} +{"id":123,"first_name":"Mildred","last_name":"Adams","email":"madamsm@usatoday.com"} +{"id":124,"first_name":"David","last_name":"Graham","email":"dgrahamn@zimbio.com"} +{"id":125,"first_name":"Victor","last_name":"Hunter","email":"vhuntero@ehow.com"} +{"id":126,"first_name":"Aaron","last_name":"Ruiz","email":"aruizp@weebly.com"} +{"id":127,"first_name":"Benjamin","last_name":"Brooks","email":"bbrooksq@jalbum.net"} +{"id":128,"first_name":"Lisa","last_name":"Wilson","email":"lwilsonr@japanpost.jp"} +{"id":129,"first_name":"Benjamin","last_name":"King","email":"bkings@comsenz.com"} +{"id":130,"first_name":"Christina","last_name":"Williamson","email":"cwilliamsont@boston.com"} +{"id":131,"first_name":"Jane","last_name":"Gonzalez","email":"jgonzalezu@networksolutions.com"} +{"id":132,"first_name":"Thomas","last_name":"Owens","email":"towensv@psu.edu"} +{"id":133,"first_name":"Katherine","last_name":"Moore","email":"kmoorew@naver.com"} +{"id":134,"first_name":"Jennifer","last_name":"Stewart","email":"jstewartx@yahoo.com"} +{"id":135,"first_name":"Sara","last_name":"Tucker","email":"stuckery@topsy.com"} +{"id":136,"first_name":"Harold","last_name":"Ortiz","email":"hortizz@vkontakte.ru"} +{"id":137,"first_name":"Shirley","last_name":"James","email":"sjames10@yelp.com"} +{"id":138,"first_name":"Dennis","last_name":"Johnson","email":"djohnson11@slate.com"} +{"id":139,"first_name":"Louise","last_name":"Weaver","email":"lweaver12@china.com.cn"} +{"id":140,"first_name":"Maria","last_name":"Armstrong","email":"marmstrong13@prweb.com"} +{"id":141,"first_name":"Gloria","last_name":"Cruz","email":"gcruz14@odnoklassniki.ru"} +{"id":142,"first_name":"Diana","last_name":"Spencer","email":"dspencer15@ifeng.com"} +{"id":143,"first_name":"Kelly","last_name":"Nguyen","email":"knguyen16@altervista.org"} +{"id":144,"first_name":"Jane","last_name":"Rodriguez","email":"jrodriguez17@biblegateway.com"} +{"id":145,"first_name":"Scott","last_name":"Brown","email":"sbrown18@geocities.jp"} +{"id":146,"first_name":"Norma","last_name":"Cruz","email":"ncruz19@si.edu"} +{"id":147,"first_name":"Marie","last_name":"Peters","email":"mpeters1a@mlb.com"} +{"id":148,"first_name":"Lillian","last_name":"Carr","email":"lcarr1b@typepad.com"} +{"id":149,"first_name":"Judy","last_name":"Nichols","email":"jnichols1c@t-online.de"} +{"id":150,"first_name":"Billy","last_name":"Long","email":"blong1d@yahoo.com"} \ No newline at end of file diff --git a/integration_tests/public_data/json/section=d/people_d.json b/integration_tests/public_data/json/section=d/people_d.json index d7512056..e7f6d3a3 100644 --- a/integration_tests/public_data/json/section=d/people_d.json +++ b/integration_tests/public_data/json/section=d/people_d.json @@ -1,300 +1,50 @@ -{ - "id": 151, - "first_name": "Howard", - "last_name": "Reid", - "email": "hreid1e@exblog.jp" -} -{ - "id": 152, - "first_name": "Laura", - "last_name": "Ferguson", - "email": "lferguson1f@tuttocitta.it" -} -{ - "id": 153, - "first_name": "Anne", - "last_name": "Bailey", - "email": "abailey1g@geocities.com" -} -{ - "id": 154, - "first_name": "Rose", - "last_name": "Morgan", - "email": "rmorgan1h@ehow.com" -} -{ - "id": 155, - "first_name": "Nicholas", - "last_name": "Reyes", - "email": "nreyes1i@google.ru" -} -{ - "id": 156, - "first_name": "Joshua", - "last_name": "Kennedy", - "email": "jkennedy1j@house.gov" -} -{ - "id": 157, - "first_name": "Paul", - "last_name": "Watkins", - "email": "pwatkins1k@upenn.edu" -} -{ - "id": 158, - "first_name": "Kathryn", - "last_name": "Kelly", - "email": "kkelly1l@businessweek.com" -} -{ - "id": 159, - "first_name": "Adam", - "last_name": "Armstrong", - "email": "aarmstrong1m@techcrunch.com" -} -{ - "id": 160, - "first_name": "Norma", - "last_name": "Wallace", - "email": "nwallace1n@phoca.cz" -} -{ - "id": 161, - "first_name": "Timothy", - "last_name": "Reyes", - "email": "treyes1o@google.cn" -} -{ - "id": 162, - "first_name": "Elizabeth", - "last_name": "Patterson", - "email": "epatterson1p@sun.com" -} -{ - "id": 163, - "first_name": "Edward", - "last_name": "Gomez", - "email": "egomez1q@google.fr" -} -{ - "id": 164, - "first_name": "David", - "last_name": "Cox", - "email": "dcox1r@friendfeed.com" -} -{ - "id": 165, - "first_name": "Brenda", - "last_name": "Wood", - "email": "bwood1s@over-blog.com" -} -{ - "id": 166, - "first_name": "Adam", - "last_name": "Walker", - "email": "awalker1t@blogs.com" -} -{ - "id": 167, - "first_name": "Michael", - "last_name": "Hart", - "email": "mhart1u@wix.com" -} -{ - "id": 168, - "first_name": "Jesse", - "last_name": "Ellis", - "email": "jellis1v@google.co.uk" -} -{ - "id": 169, - "first_name": "Janet", - "last_name": "Powell", - "email": "jpowell1w@un.org" -} -{ - "id": 170, - "first_name": "Helen", - "last_name": "Ford", - "email": "hford1x@creativecommons.org" -} -{ - "id": 171, - "first_name": "Gerald", - "last_name": "Carpenter", - "email": "gcarpenter1y@about.me" -} -{ - "id": 172, - "first_name": "Kathryn", - "last_name": "Oliver", - "email": "koliver1z@army.mil" -} -{ - "id": 173, - "first_name": "Alan", - "last_name": "Berry", - "email": "aberry20@gov.uk" -} -{ - "id": 174, - "first_name": "Harry", - "last_name": "Andrews", - "email": "handrews21@ameblo.jp" -} -{ - "id": 175, - "first_name": "Andrea", - "last_name": "Hall", - "email": "ahall22@hp.com" -} -{ - "id": 176, - "first_name": "Barbara", - "last_name": "Wells", - "email": "bwells23@behance.net" -} -{ - "id": 177, - "first_name": "Anne", - "last_name": "Wells", - "email": "awells24@apache.org" -} -{ - "id": 178, - "first_name": "Harry", - "last_name": "Harper", - "email": "hharper25@rediff.com" -} -{ - "id": 179, - "first_name": "Jack", - "last_name": "Ray", - "email": "jray26@wufoo.com" -} -{ - "id": 180, - "first_name": "Phillip", - "last_name": "Hamilton", - "email": "phamilton27@joomla.org" -} -{ - "id": 181, - "first_name": "Shirley", - "last_name": "Hunter", - "email": "shunter28@newsvine.com" -} -{ - "id": 182, - "first_name": "Arthur", - "last_name": "Daniels", - "email": "adaniels29@reuters.com" -} -{ - "id": 183, - "first_name": "Virginia", - "last_name": "Rodriguez", - "email": "vrodriguez2a@walmart.com" -} -{ - "id": 184, - "first_name": "Christina", - "last_name": "Ryan", - "email": "cryan2b@hibu.com" -} -{ - "id": 185, - "first_name": "Theresa", - "last_name": "Mendoza", - "email": "tmendoza2c@vinaora.com" -} -{ - "id": 186, - "first_name": "Jason", - "last_name": "Cole", - "email": "jcole2d@ycombinator.com" -} -{ - "id": 187, - "first_name": "Phillip", - "last_name": "Bryant", - "email": "pbryant2e@rediff.com" -} -{ - "id": 188, - "first_name": "Adam", - "last_name": "Torres", - "email": "atorres2f@sun.com" -} -{ - "id": 189, - "first_name": "Margaret", - "last_name": "Johnston", - "email": "mjohnston2g@ucsd.edu" -} -{ - "id": 190, - "first_name": "Paul", - "last_name": "Payne", - "email": "ppayne2h@hhs.gov" -} -{ - "id": 191, - "first_name": "Todd", - "last_name": "Willis", - "email": "twillis2i@businessweek.com" -} -{ - "id": 192, - "first_name": "Willie", - "last_name": "Oliver", - "email": "woliver2j@noaa.gov" -} -{ - "id": 193, - "first_name": "Frances", - "last_name": "Robertson", - "email": "frobertson2k@go.com" -} -{ - "id": 194, - "first_name": "Gregory", - "last_name": "Hawkins", - "email": "ghawkins2l@joomla.org" -} -{ - "id": 195, - "first_name": "Lisa", - "last_name": "Perkins", - "email": "lperkins2m@si.edu" -} -{ - "id": 196, - "first_name": "Jacqueline", - "last_name": "Anderson", - "email": "janderson2n@cargocollective.com" -} -{ - "id": 197, - "first_name": "Shirley", - "last_name": "Diaz", - "email": "sdiaz2o@ucla.edu" -} -{ - "id": 198, - "first_name": "Nicole", - "last_name": "Meyer", - "email": "nmeyer2p@flickr.com" -} -{ - "id": 199, - "first_name": "Mary", - "last_name": "Gray", - "email": "mgray2q@constantcontact.com" -} -{ - "id": 200, - "first_name": "Jean", - "last_name": "Mcdonald", - "email": "jmcdonald2r@baidu.com" -} \ No newline at end of file +{"id":151,"first_name":"Howard","last_name":"Reid","email":"hreid1e@exblog.jp"} +{"id":152,"first_name":"Laura","last_name":"Ferguson","email":"lferguson1f@tuttocitta.it"} +{"id":153,"first_name":"Anne","last_name":"Bailey","email":"abailey1g@geocities.com"} +{"id":154,"first_name":"Rose","last_name":"Morgan","email":"rmorgan1h@ehow.com"} +{"id":155,"first_name":"Nicholas","last_name":"Reyes","email":"nreyes1i@google.ru"} +{"id":156,"first_name":"Joshua","last_name":"Kennedy","email":"jkennedy1j@house.gov"} +{"id":157,"first_name":"Paul","last_name":"Watkins","email":"pwatkins1k@upenn.edu"} +{"id":158,"first_name":"Kathryn","last_name":"Kelly","email":"kkelly1l@businessweek.com"} +{"id":159,"first_name":"Adam","last_name":"Armstrong","email":"aarmstrong1m@techcrunch.com"} +{"id":160,"first_name":"Norma","last_name":"Wallace","email":"nwallace1n@phoca.cz"} +{"id":161,"first_name":"Timothy","last_name":"Reyes","email":"treyes1o@google.cn"} +{"id":162,"first_name":"Elizabeth","last_name":"Patterson","email":"epatterson1p@sun.com"} +{"id":163,"first_name":"Edward","last_name":"Gomez","email":"egomez1q@google.fr"} +{"id":164,"first_name":"David","last_name":"Cox","email":"dcox1r@friendfeed.com"} +{"id":165,"first_name":"Brenda","last_name":"Wood","email":"bwood1s@over-blog.com"} +{"id":166,"first_name":"Adam","last_name":"Walker","email":"awalker1t@blogs.com"} +{"id":167,"first_name":"Michael","last_name":"Hart","email":"mhart1u@wix.com"} +{"id":168,"first_name":"Jesse","last_name":"Ellis","email":"jellis1v@google.co.uk"} +{"id":169,"first_name":"Janet","last_name":"Powell","email":"jpowell1w@un.org"} +{"id":170,"first_name":"Helen","last_name":"Ford","email":"hford1x@creativecommons.org"} +{"id":171,"first_name":"Gerald","last_name":"Carpenter","email":"gcarpenter1y@about.me"} +{"id":172,"first_name":"Kathryn","last_name":"Oliver","email":"koliver1z@army.mil"} +{"id":173,"first_name":"Alan","last_name":"Berry","email":"aberry20@gov.uk"} +{"id":174,"first_name":"Harry","last_name":"Andrews","email":"handrews21@ameblo.jp"} +{"id":175,"first_name":"Andrea","last_name":"Hall","email":"ahall22@hp.com"} +{"id":176,"first_name":"Barbara","last_name":"Wells","email":"bwells23@behance.net"} +{"id":177,"first_name":"Anne","last_name":"Wells","email":"awells24@apache.org"} +{"id":178,"first_name":"Harry","last_name":"Harper","email":"hharper25@rediff.com"} +{"id":179,"first_name":"Jack","last_name":"Ray","email":"jray26@wufoo.com"} +{"id":180,"first_name":"Phillip","last_name":"Hamilton","email":"phamilton27@joomla.org"} +{"id":181,"first_name":"Shirley","last_name":"Hunter","email":"shunter28@newsvine.com"} +{"id":182,"first_name":"Arthur","last_name":"Daniels","email":"adaniels29@reuters.com"} +{"id":183,"first_name":"Virginia","last_name":"Rodriguez","email":"vrodriguez2a@walmart.com"} +{"id":184,"first_name":"Christina","last_name":"Ryan","email":"cryan2b@hibu.com"} +{"id":185,"first_name":"Theresa","last_name":"Mendoza","email":"tmendoza2c@vinaora.com"} +{"id":186,"first_name":"Jason","last_name":"Cole","email":"jcole2d@ycombinator.com"} +{"id":187,"first_name":"Phillip","last_name":"Bryant","email":"pbryant2e@rediff.com"} +{"id":188,"first_name":"Adam","last_name":"Torres","email":"atorres2f@sun.com"} +{"id":189,"first_name":"Margaret","last_name":"Johnston","email":"mjohnston2g@ucsd.edu"} +{"id":190,"first_name":"Paul","last_name":"Payne","email":"ppayne2h@hhs.gov"} +{"id":191,"first_name":"Todd","last_name":"Willis","email":"twillis2i@businessweek.com"} +{"id":192,"first_name":"Willie","last_name":"Oliver","email":"woliver2j@noaa.gov"} +{"id":193,"first_name":"Frances","last_name":"Robertson","email":"frobertson2k@go.com"} +{"id":194,"first_name":"Gregory","last_name":"Hawkins","email":"ghawkins2l@joomla.org"} +{"id":195,"first_name":"Lisa","last_name":"Perkins","email":"lperkins2m@si.edu"} +{"id":196,"first_name":"Jacqueline","last_name":"Anderson","email":"janderson2n@cargocollective.com"} +{"id":197,"first_name":"Shirley","last_name":"Diaz","email":"sdiaz2o@ucla.edu"} +{"id":198,"first_name":"Nicole","last_name":"Meyer","email":"nmeyer2p@flickr.com"} +{"id":199,"first_name":"Mary","last_name":"Gray","email":"mgray2q@constantcontact.com"} +{"id":200,"first_name":"Jean","last_name":"Mcdonald","email":"jmcdonald2r@baidu.com"} \ No newline at end of file diff --git a/macros/plugins/athena/create_external_table.sql b/macros/plugins/athena/create_external_table.sql new file mode 100644 index 00000000..636eff9b --- /dev/null +++ b/macros/plugins/athena/create_external_table.sql @@ -0,0 +1,33 @@ +{% macro athena__create_external_table(source_node) %} + {%- set columns = source_node.columns.values() -%} + {%- set external = source_node.external -%} + {# https://docs.aws.amazon.com/athena/latest/ug/create-table.html #} + create external table {{source(source_node.source_name, source_node.name)}} ( + {% for column in columns %} + {{column.name}} {{column.data_type}} + {{- ',' if not loop.last -}} + {% endfor %} + ) + {% if external.comment -%} comment '{{external.comment}}' {%- endif %} + {% if external.partitions -%} + partitioned by ( + {% for partition in external.partitions %} + {{partition.name}} {{partition.data_type}} + {%- if partition.comment %} comment '{{partition.comment}}' {%- endif -%} + {{- ', ' if not loop.last -}} + {% endfor %} + ) + {%- endif %} + {% if external.clusters and external.num_buckets -%} + clustered by ( + {%- for column in external.clusters -%} + {{column}}{{', ' if not loop.last}} + {%- endfor -%} + ) into num_buckets {{external.num_buckets}} + {%- endif %} + {% if external.row_format -%} row format {{external.row_format}} {%- endif %} + {% if external.file_format -%} stored as {{external.file_format}} {%- endif %} + {% if external.serde_properties -%} with serdeproperties {{external.serde_properties}} {%- endif %} + {% if external.location -%} location '{{external.location}}' {%- endif %} + {% if external.table_properties -%} tblproperties {{external.table_properties}} {%- endif %} +{% endmacro %} diff --git a/macros/plugins/athena/get_external_build_plan.sql b/macros/plugins/athena/get_external_build_plan.sql new file mode 100644 index 00000000..0d06b752 --- /dev/null +++ b/macros/plugins/athena/get_external_build_plan.sql @@ -0,0 +1,19 @@ +{% macro athena__get_external_build_plan(source_node) %} + {% set build_plan = [] %} + {% set old_relation = adapter.get_relation( + database = source_node.database, + schema = source_node.schema, + identifier = source_node.identifier + ) %} + {% set create_or_replace = (old_relation is none or var('ext_full_refresh', false)) %} + {% if create_or_replace %} + {% set build_plan = [ + dbt_external_tables.dropif(source_node), + dbt_external_tables.create_external_table(source_node) + ] + dbt_external_tables.refresh_external_table(source_node) + %} + {% else %} + {% set build_plan = dbt_external_tables.refresh_external_table(source_node) %} + {% endif %} + {% do return(build_plan) %} +{% endmacro %} diff --git a/macros/plugins/athena/helpers/dropif.sql b/macros/plugins/athena/helpers/dropif.sql new file mode 100644 index 00000000..a0cc518c --- /dev/null +++ b/macros/plugins/athena/helpers/dropif.sql @@ -0,0 +1,6 @@ +{% macro athena__dropif(node) %} + {% set ddl %} + drop table if exists {{source(node.source_name, node.name)}} + {% endset %} + {{return(ddl)}} +{% endmacro %} diff --git a/macros/plugins/athena/refresh_external_tables.sql b/macros/plugins/athena/refresh_external_tables.sql new file mode 100644 index 00000000..c1d1c832 --- /dev/null +++ b/macros/plugins/athena/refresh_external_tables.sql @@ -0,0 +1,61 @@ +{% macro athena__refresh_external_table(source_node) %} + {# https://docs.aws.amazon.com/athena/latest/ug/partitions.html #} + {%- set partitions = source_node.external.partitions -%} + {%- set hive_compatible_partitions = source_node.external.get('hive_compatible_partitions', false) -%} + {%- if partitions -%} + {%- if hive_compatible_partitions -%} + {% set ddl -%} + msck repair table {{source(source_node.source_name, source_node.name)}} + {%- endset %} + {{ return([ddl]) }} + {% else %} + {# https://docs.aws.amazon.com/athena/latest/ug/alter-table-add-partition.html #} + {%- set part_len = partitions|length -%} + {%- set get_partitions_sql -%} + select * from + {%- for partition in partitions %} ( + {%- set part_num = loop.index -%} + {%- if partition.vals.macro -%} + {%- set vals = dbt_external_tables.render_from_context(partition.vals.macro, **partition.vals.args) -%} + {%- elif partition.vals is string -%} + {%- set vals = [partition.vals] -%} + {%- else -%} + {%- set vals = partition.vals -%} + {%- endif -%} + {%- for val in vals %} + select + '"{{ partition.name }}"' as name_{{ part_num }}, + '"{{ val }}"' as val_{{ part_num }}, + '"{{ dbt_external_tables.render_from_context(partition.path_macro, partition.name, val) }}"' as path_{{ part_num }} + {{ 'union all' if not loop.last else ') ' }} + {%- endfor -%} + {{ 'cross join' if not loop.last }} + {%- endfor -%} + {%- endset -%} + {%- set finals = [] -%} + {%- if execute -%} + {%- set results = run_query(get_partitions_sql) -%} + {%- for row in results -%} + {%- set partition_parts = [] -%} + {%- set path_parts = [] -%} + {%- for i in range(0, part_len) -%} + {%- do partition_parts.append({ + 'name': row[i * 3][1:-1], + 'value': row[i * 3 + 1][1:-1] + }) -%} + {%- do path_parts.append(row[i * 3 + 2][1:-1]) -%} + {%- endfor -%} + {%- set construct = { + 'partition_by': partition_parts, + 'path': path_parts | join('/') + } -%} + {% do finals.append(construct) %} + {%- endfor -%} + {%- endif -%} + {%- set ddl = dbt_external_tables.redshift_alter_table_add_partitions(source_node, finals) -%} + {{ return(ddl) }} + {% do return([]) %} + {% endif %} + {% endif %} + {% do return([]) %} +{% endmacro %} diff --git a/run_test.sh b/run_test.sh index f6be5a64..e8609d47 100755 --- a/run_test.sh +++ b/run_test.sh @@ -14,6 +14,10 @@ if [[ ! -f $VENV ]]; then then echo "Installing dbt-sqlserver" pip install dbt-sqlserver --upgrade --pre + elif [ $1 == 'athena' ] + then + echo "Installing dbt-athena" + pip install git+https://github.com/Tomme/dbt-athena.git --upgrade --pre else echo "Installing dbt-$1" pip install dbt-$1 --upgrade --pre diff --git a/sample_sources/athena.yml b/sample_sources/athena.yml new file mode 100644 index 00000000..f115bed0 --- /dev/null +++ b/sample_sources/athena.yml @@ -0,0 +1,61 @@ +version: 2 + +sources: + - name: dbt + tables: + - name: hrsl + database: awscatalog + schema: dbt + external: + location: 's3://dataforgood-fb-data/csv/' + row_format: "SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'" + serde_properties: "('serialization.format' = '\t\t', 'field.delim' = '\t')" + table_properties: ('has_encrypted_data'='false') + hive_compatible_partitions: true + partitions: + - name: month + data_type: string + - name: country + data_type: string + - name: type + data_type: string + columns: + - name: latitude + data_type: float + - name: longitude + data_type: float + - name: population + data_type: float + + - name: planet_history + database: awscatalog + schema: dbt + external: + location: 's3://osm-pds/planet/' + file_format: ORCFILE + comment: "This is a table comment" + columns: + - name: id + data_type: 'BIGINT' + - name: type + data_type: 'STRING' + - name: tags + data_type: 'MAP' + - name: lat + data_type: 'DECIMAL(9,7)' + - name: lon + data_type: 'DECIMAL(10,7)' + - name: nds + data_type: 'ARRAY>' + - name: members + data_type: 'ARRAY>' + - name: changeset + data_type: 'BIGINT' + - name: timestamp + data_type: 'TIMESTAMP' + - name: uid + data_type: 'BIGINT' + - name: user + data_type: 'STRING' + - name: version + data_type: 'BIGINT'