From 51afc406c22981974611ec03a2df36518a23a8d7 Mon Sep 17 00:00:00 2001 From: Tim Schaub Date: Mon, 18 Nov 2024 15:42:49 -0700 Subject: [PATCH] Add test with bad newline-delimited data --- cmd/gpq/command/convert.go | 36 +++++++++++++------ internal/geojson/featurereader_test.go | 14 ++++++++ .../testdata/bad-new-line-delimited.ndgeojson | 2 ++ 3 files changed, 42 insertions(+), 10 deletions(-) create mode 100644 internal/geojson/testdata/bad-new-line-delimited.ndgeojson diff --git a/cmd/gpq/command/convert.go b/cmd/gpq/command/convert.go index ab90aa2..febb425 100644 --- a/cmd/gpq/command/convert.go +++ b/cmd/gpq/command/convert.go @@ -17,7 +17,8 @@ package command import ( "net/url" "os" - "regexp" + "path/filepath" + "slices" "strings" "github.com/planetlabs/gpq/internal/geojson" @@ -53,8 +54,6 @@ var validTypes = map[FormatType]bool{ GeoJSONType: true, } -var geoJsonMatcher *regexp.Regexp - func parseFormatType(format string) FormatType { if format == "" { return AutoType @@ -66,19 +65,38 @@ func parseFormatType(format string) FormatType { return ft } +var geoParquetSuffixes = []string{ + ".gpq", ".geoparquet", +} + +var parquetSuffixes = []string{ + ".pq", ".parquet", +} + +var geoJsonSuffixes = []string{ + ".geojson", + ".json", + ".ndjson", + ".ndgeojson", + ".geojsonl", +} + func getFormatType(resource string) FormatType { if u, err := url.Parse(resource); err == nil { resource = u.Path } - if geoJsonMatcher.MatchString(resource) { - return GeoJSONType - } - if strings.HasSuffix(resource, ".gpq") || strings.HasSuffix(resource, ".geoparquet") { + + ext := filepath.Ext(resource) + if slices.Contains(geoParquetSuffixes, ext) { return GeoParquetType } - if strings.HasSuffix(resource, ".pq") || strings.HasSuffix(resource, ".parquet") { + if slices.Contains(parquetSuffixes, ext) { return ParquetType } + if slices.Contains(geoJsonSuffixes, ext) { + return GeoJSONType + } + return UnknownType } @@ -99,8 +117,6 @@ func (c *ConvertCmd) Run() error { inputSource = "" } - geoJsonMatcher = regexp.MustCompile(`.+\.(geojson|json|ndjson|ndgeojson|geojsonl)$`) - outputFormat := parseFormatType(c.To) if outputFormat == AutoType { if outputSource == "" { diff --git a/internal/geojson/featurereader_test.go b/internal/geojson/featurereader_test.go index 169cdb1..31f4ed4 100644 --- a/internal/geojson/featurereader_test.go +++ b/internal/geojson/featurereader_test.go @@ -120,6 +120,20 @@ func TestFeatureReaderNewLineDelimited(t *testing.T) { assert.Equal(t, float64(326625791), usa.Properties["pop_est"]) } +func TestFeatureReaderBadNewLineDelimited(t *testing.T) { + file, openErr := os.Open("testdata/bad-new-line-delimited.ndgeojson") + require.NoError(t, openErr) + + reader := geojson.NewFeatureReader(file) + + first, err := reader.Read() + require.NoError(t, err) + assert.Equal(t, "Oceania", first.Properties["continent"]) + + _, err = reader.Read() + assert.ErrorContains(t, err, "unexpected end of JSON input") +} + func TestFeatureReaderEmptyFeatureCollection(t *testing.T) { file, openErr := os.Open("testdata/empty-collection.geojson") require.NoError(t, openErr) diff --git a/internal/geojson/testdata/bad-new-line-delimited.ndgeojson b/internal/geojson/testdata/bad-new-line-delimited.ndgeojson new file mode 100644 index 0000000..2421783 --- /dev/null +++ b/internal/geojson/testdata/bad-new-line-delimited.ndgeojson @@ -0,0 +1,2 @@ +{"geometry": {"type": "MultiPolygon", "coordinates": [[[[180, -16.067132663642447], [180, -16.555216566639196], [179.36414266196414, -16.801354076946883], [178.72505936299711, -17.01204167436804], [178.59683859511713, -16.639150000000004], [179.0966093629971, -16.433984277547403], [179.4135093629971, -16.379054277547404], [180, -16.067132663642447]]], [[[178.12557, -17.50481], [178.3736, -17.33992], [178.71806, -17.62846], [178.55271, -18.15059], [177.93266000000003, -18.28799], [177.38146, -18.16432], [177.28504, -17.72465], [177.67087, -17.381140000000002], [178.12557, -17.50481]]], [[[-179.79332010904864, -16.020882256741224], [-179.9173693847653, -16.501783135649397], [-180, -16.555216566639196], [-180, -16.067132663642447], [-179.79332010904864, -16.020882256741224]]]]}, "properties": {"continent": "Oceania", "gdp_md_est": 8374, "iso_a3": "FJI", "name": "Fiji", "pop_est": 920938}, "type": "Feature"} +{"type": "MultiPolygon", "coordinates": [[[[-122.84000000000003, 49.000000000000114], [-120, 49.000000000000114], [-117.03121, 49], [-116.04818, 49], [-113, 49], [-110.05000000000001, 49], [-107.05000000000001, 49], [-104.04826000000003, 48.99986000000007], [-100.65000000000003, 49.000000000000114], [-97.2287200000048, 49.0007], [-95.15906950917206, 49], [-95.15609, 49.38425000000001], [-94.81758000000002, 49.38905], [-94.64, 48.84], [-94.32914000000001, 48.67074], [-93.63087000000002, 48.609260000000006], [-92.61000000000001, 48.44999999999993], [-91.64, 48.14], [-90.83, 48.27], [-89.60000000000002, 48.010000000000105], [-89.27291744663665, 48.01980825458281], [-88.37811418328671, 48.302917588893706], [-87.43979262330028, 47.94], [-86.46199083122826, 47.553338019392], [-85.65236324740341, 47.22021881773051], [-84.8760798815149, 46.90008331968238], [-84.77923824739992, 46.63710195574902], [-84.54374874544584, 46.538684190449146], [-84.60490000000004, 46.439599999999984], [-84.33670000000001, 46.408770000000004], [-84.1421195136734, 46.51222585711571], [-84.0918512641615, 46.27541860613826], [-83.89076534700574, 46.116926988299014], [-83.61613094759059, 46.116926988299014], [-83.46955074739469, 45.994686387712534], [-83.59285071484311, 45.81689362241252], [-82.55092464875821, 45.34751658790543], [-82.33776312543114, 44.440000000000055], [-82.13764238150395, 43.57108755143997], [-82.42999999999999, 42.980000000000004], [-82.9, 42.43], [-83.12, 42.08], [-83.14199968131264, 41.975681057292874], [-83.029810146807, 41.83279572200598], [-82.69008928092023, 41.675105088867326], [-82.4392777167916, 41.675105088867326], [-81.27774654816716, 42.209025987306816], [-80.24744767934794, 42.36619985612255], [-78.93936214874375, 42.86361135514798], [-78.92, 42.964999999999996], [-79.01, 43.27], [-79.17167355011186, 43.46633942318426], [-78.72027991404235, 43.62508942318493], [-77.7378850979577, 43.62905558936328], [-76.82003414580558, 43.628784288093755], [-76.50000000000001, 44.01845889375865], [-76.375, 44.09631], [-75.31821000000001, 44.81645], [-74.86700000000002, 45.000480000000096], [-73.34783, 45.00738], [-71.50506, 45.0082], [-71.405, 45.254999999999995], [-71.08482000000004, 45.30524000000014], [-70.66, 45.46], [-70.305, 45.915], [-69.99997, 46.69307], [-69.237216, 47.447781], [-68.90500000000003, 47.18500000000006], [-68.23444, 47.354859999999974], [-67.79046000000001, 47.066359999999996], [-67.79134, 45.70281000000001], [-67.13741, 45.13753], [-66.96465999999998, 44.809700000000134], [-68.03251999999998, 44.325199999999995], [-69.05999999999995, 43.980000000000075], [-70.11616999999995, 43.68405000000013], [-70.64547563341102, 43.09023834896402], [-70.81488999999999, 42.865299999999934], [-70.82499999999999, 42.33499999999998], [-70.49499999999995, 41.80500000000001], [-70.07999999999998, 41.78000000000003], [-70.185, 42.145000000000095], [-69.88496999999995, 41.92283000000009], [-69.96502999999996, 41.63717000000014], [-70.63999999999999, 41.47500000000002], [-71.12039000000004, 41.49445000000014], [-71.8599999999999, 41.32000000000005], [-72.29500000000002, 41.26999999999998], [-72.87643000000003, 41.220650000000035], [-73.71000000000004, 40.93110235165449], [-72.24125999999995, 41.119480000000124], [-71.94499999999988, 40.930000000000064], [-73.34499999999997, 40.63000000000005], [-73.98200000000003, 40.62799999999993], [-73.95232499999997, 40.75075000000004], [-74.25671, 40.47351000000003], [-73.96243999999996, 40.42763000000002], [-74.17838, 39.70925999999997], [-74.90603999999996, 38.93954000000002], [-74.98041, 39.19640000000004], [-75.20002, 39.248450000000105], [-75.52805000000001, 39.49850000000009], [-75.32, 38.960000000000036], [-75.07183476478986, 38.782032230179254], [-75.05672999999996, 38.40412000000009], [-75.37746999999996, 38.015510000000006], [-75.94022999999999, 37.21689000000009], [-76.03126999999995, 37.25659999999999], [-75.72204999999985, 37.93705000000011], [-76.23286999999999, 38.319214999999986], [-76.35000000000002, 39.14999999999998], [-76.54272499999996, 38.71761500000008], [-76.32933000000003, 38.08326000000005], [-76.98999793161352, 38.23999176691336], [-76.30161999999996, 37.91794499999992], [-76.25873999999999, 36.96640000000008], [-75.97179999999997, 36.89726000000002], [-75.8680399999999, 36.55125000000004], [-75.72748999999999, 35.55074000000013], [-76.36318, 34.80854000000011], [-77.39763499999992, 34.512009999999975], [-78.05496, 33.92547000000002], [-78.55434999999989, 33.86133000000012], [-79.06067000000002, 33.493949999999984], [-79.20357000000001, 33.158390000000054], [-80.30132499999996, 32.509355000000085], [-80.86498, 32.033300000000054], [-81.33629000000002, 31.44049000000001], [-81.49041999999997, 30.7299900000001], [-81.31371000000001, 30.035520000000076], [-80.97999999999996, 29.18000000000012], [-80.53558499999991, 28.472129999999993], [-80.52999999999986, 28.040000000000077], [-80.05653928497759, 26.88000000000011], [-80.08801499999998, 26.205764999999985], [-80.13155999999992, 25.816775000000064], [-80.38103000000001, 25.20616000000001], [-80.67999999999995, 25.08000000000004], [-81.17212999999998, 25.201260000000104], [-81.33000000000004, 25.639999999999986], [-81.70999999999987, 25.870000000000005], [-82.23999999999995, 26.730000000000132], [-82.70515, 27.495040000000074], [-82.85525999999999, 27.886240000000043], [-82.64999999999998, 28.550000000000125], [-82.92999999999995, 29.10000000000008], [-83.70958999999999, 29.936560000000043], [-84.09999999999997, 30.09000000000009], [-85.10881999999998, 29.636150000000043], [-85.28784000000002, 29.68612000000013], [-85.7731, 30.152610000000095], [-86.39999999999992, 30.40000000000009], [-87.53035999999992, 30.27433000000002], [-88.41781999999995, 30.384900000000016], [-89.1804899999999, 30.315980000000025], [-89.5938311784198, 30.159994004836847], [-89.41373499999997, 29.89418999999998], [-89.43, 29.488639999999975], [-89.21767, 29.291080000000022], [-89.40822999999995, 29.159610000000043], [-89.77927999999997, 29.307140000000118], [-90.15463, 29.11743000000007], [-90.88022499999994, 29.148535000000095], [-91.62678499999993, 29.677000000000135], [-92.49905999999999, 29.552300000000002], [-93.22636999999997, 29.783750000000055], [-93.84841999999998, 29.71363000000008], [-94.69, 29.480000000000132], [-95.60025999999999, 28.738630000000057], [-96.59403999999995, 28.307480000000055], [-97.13999999999987, 27.83000000000004], [-97.36999999999995, 27.380000000000052], [-97.37999999999994, 26.690000000000055], [-97.32999999999998, 26.210000000000093], [-97.13999999999987, 25.870000000000005], [-97.52999999999992, 25.84000000000009], [-98.23999999999995, 26.06000000000006], [-99.01999999999992, 26.37000000000006], [-99.30000000000001, 26.840000000000032], [-99.51999999999992, 27.54000000000002], [-100.10999999999996, 28.110000000000127], [-100.45584000000002, 28.69612000000012], [-100.95759999999996, 29.380710000000136], [-101.66239999999999, 29.77930000000009], [-102.48000000000002, 29.75999999999999], [-103.11000000000001, 28.970000000000027], [-103.94, 29.27000000000004], [-104.4569699999999, 29.571960000000047], [-104.70574999999997, 30.121730000000014], [-105.03737000000001, 30.644019999999955], [-105.63159000000002, 31.08383000000009], [-106.1429, 31.399950000000047], [-106.50758999999988, 31.754520000000014], [-108.24000000000001, 31.754853718166373], [-108.24193999999994, 31.342220000000054], [-109.03500000000003, 31.341940000000136], [-111.02361000000002, 31.334719999999948], [-113.30498, 32.03914000000009], [-114.815, 32.52528000000001], [-114.72138999999993, 32.72082999999992], [-115.99134999999995, 32.61239000000012], [-117.12775999999985, 32.53533999999996], [-117.29593769127393, 33.04622461520387], [-117.94400000000002, 33.621236431201396], [-118.41060227589753, 33.74090922312445], [-118.51989482279976, 34.02778157757575], [-119.08100000000002, 34.07799999999992], [-119.43884064201671, 34.34847717828427], [-120.36777999999998, 34.447110000000066], [-120.62286, 34.60854999999998], [-120.74432999999999, 35.15686000000011], [-121.71456999999992, 36.161529999999914], [-122.54746999999998, 37.551760000000115], [-122.51201000000003, 37.78339000000011], [-122.95319, 38.11371000000008], [-123.72720000000004, 38.95166000000012], [-123.86516999999998, 39.76699000000008], [-124.39807000000002, 40.313199999999995], [-124.17885999999999, 41.142020000000116], [-124.21370000000002, 41.99964000000011], [-124.53283999999996, 42.7659900000001], [-124.14213999999998, 43.708380000000034], [-124.020535, 44.615894999999966], [-123.89892999999995, 45.52341000000007], [-124.079635, 46.864750000000015], [-124.39566999999994, 47.72017000000011], [-124.68721008300781, 48.18443298339855], [-124.56610107421875, 48.37971496582037], [-123.12, 48.04000000000002], [-122.58735999999993, 47.09600000000006], [-122.34000000000003, 47.360000000000014], [-122.5, 48.180000000000064], [-122.84000000000003, 49.000000000000114]]], [[[-155.40214, 20.07975], [-155.22452, 19.99302], [-155.06226, 19.8591], [-154.80741, 19.50871], [-154.83147, 19.453280000000003], [-155.22217, 19.23972], [-155.54211, 19.08348], [-155.68817, 18.91619], [-155.93665, 19.05939], [-155.90806, 19.33888], [-156.07347000000001, 19.70294], [-156.02368, 19.81422], [-155.85008000000002, 19.97729], [-155.91907, 20.17395], [-155.86108000000002, 20.267210000000002], [-155.78505, 20.2487], [-155.40214, 20.07975]]], [[[-155.99566000000002, 20.76404], [-156.07926, 20.643970000000003], [-156.41445, 20.57241], [-156.58673, 20.783], [-156.70167, 20.8643], [-156.71054999999998, 20.92676], [-156.61258, 21.01249], [-156.25711, 20.917450000000002], [-155.99566000000002, 20.76404]]], [[[-156.75824, 21.176840000000002], [-156.78933, 21.068730000000002], [-157.32521, 21.097770000000004], [-157.25027, 21.219579999999997], [-156.75824, 21.176840000000002]]], [[[-158.0252, 21.71696], [-157.94161, 21.65272], [-157.65283000000002, 21.322170000000003], [-157.70703, 21.26442], [-157.7786, 21.27729], [-158.12667000000002, 21.31244], [-158.2538, 21.53919], [-158.29265, 21.57912], [-158.0252, 21.71696]]], [[[-159.36569, 22.21494], [-159.34512, 21.982000000000003], [-159.46372, 21.88299], [-159.80051, 22.065330000000003], [-159.74877, 22.1382], [-159.5962, 22.236179999999997], [-159.36569, 22.21494]]], [[[-166.46779212142462, 60.384169826897754], [-165.67442969466364, 60.29360687930625], [-165.57916419173358, 59.90998688418753], [-166.19277014876727, 59.75444082298899], [-166.84833736882197, 59.941406155020985], [-167.45527706609008, 60.21306915957936], [-166.46779212142462, 60.384169826897754]]], [[[-153.22872941792113, 57.96896841087248], [-152.56479061583514, 57.901427313866996], [-152.1411472239064, 57.591058661522], [-153.00631405333692, 57.11584219016593], [-154.0050902984581, 56.734676825581076], [-154.51640275777004, 56.99274892844669], [-154.67099280497118, 57.46119578717253], [-153.7627795074415, 57.81657461204373], [-153.22872941792113, 57.96896841087248]]], [[[-140.98598761037601, 69.71199839952635], [-140.986, 69.712], [-140.9925, 66.00003000000001], [-140.99778, 60.30639000000001], [-140.013, 60.27682000000001], [-139.03900000000002, 60], [-138.34089, 59.562110000000004], [-137.4525, 58.905], [-136.47972000000004, 59.46389000000005], [-135.47583, 59.787780000000005], [-134.94500000000005, 59.2705600000001], [-134.27111000000002, 58.86111000000005], [-133.35556000000003, 58.41028000000001], [-132.73042, 57.692890000000006], [-131.70781, 56.55212], [-130.00778000000003, 55.915830000000085], [-129.98, 55.285000000000004], [-130.53611, 54.802780000000006], [-130.53610895273684, 54.80275447679924], [-130.5361101894673, 54.8027534043494], [-131.08581823797215, 55.17890615500204], [-131.9672114671423, 55.497775580459006], [-132.2500107428595, 56.3699962428974], [-133.53918108435641, 57.17888743756214], [-134.07806292029608, 58.12306753196691], [-135.0382110322791, 58.18771474876394], [-136.62806230995471, 58.21220937767043], [-137.800006279686, 58.49999542910376], [-139.867787041413, 59.53776154238915], [-140.825273817133, 59.727517401765056], [-142.57444353556446, 60.08444651960497], [-143.9588809948799, 59.999180406323376], [-145.92555681682788, 60.45860972761426], [-147.11437394914665, 60.884656073644635], [-148.22430620012761, 60.67298940697714], [-148.01806555885082, 59.97832896589364], [-148.57082251686086, 59.914172675203304], [-149.72785783587585, 59.70565827090553], [-150.60824337461642, 59.368211168039466], [-151.7163927886833, 59.15582103131993], [-151.85943315326722, 59.744984035879554], [-151.40971900124717, 60.72580272077937], [-150.3469414947325, 61.03358755150987], [-150.62111080625704, 61.2844249538544], [-151.89583919981683, 60.727197984451266], [-152.57832984109558, 60.061657212964235], [-154.01917212625764, 59.35027944603428], [-153.28751135965317, 58.86472768821977], [-154.23249243875847, 58.14637360293051], [-155.3074914215102, 57.727794501366304], [-156.30833472392305, 57.422774359763594], [-156.55609737854638, 56.97998484967064], [-158.11721655986779, 56.46360809999419], [-158.43332129619714, 55.99415355083852], [-159.60332739971741, 55.56668610292013], [-160.28971961163427, 55.643580634170576], [-161.22304765525777, 55.364734605523495], [-162.23776607974105, 55.02418691672011], [-163.06944658104638, 54.68973704692712], [-164.78556922102717, 54.40417308208214], [-164.94222632552007, 54.57222483989534], [-163.84833960676565, 55.03943146424609], [-162.87000139061595, 55.34804311789321], [-161.80417497459607, 55.89498647727038], [-160.5636047027812, 56.00805451112501], [-160.07055986228448, 56.41805532492873], [-158.6844429189195, 57.01667511659787], [-158.46109737855403, 57.21692129172885], [-157.72277035218391, 57.57000051536306], [-157.55027442119362, 58.328326321030204], [-157.04167497457698, 58.91888458926172], [-158.19473120830554, 58.61580231386978], [-158.51721798402303, 58.78778148053732], [-159.0586061269288, 58.42418610293163], [-159.71166704001737, 58.93139028587632], [-159.98128882550017, 58.572549140041644], [-160.3552711659965, 59.07112335879361], [-161.3550034251151, 58.670837714260756], [-161.96889360252632, 58.67166453717738], [-162.05498653872465, 59.26692536074745], [-161.8741707021354, 59.63362132429057], [-162.51805904849212, 59.98972361921386], [-163.8183414378202, 59.79805573184336], [-164.66221757714652, 60.26748444278263], [-165.3463877024748, 60.50749563256238], [-165.3508318756519, 61.073895168697504], [-166.12137915755602, 61.50001902937623], [-165.73445187077058, 62.074996853271784], [-164.9191786367179, 62.63307648380794], [-164.56250790103934, 63.14637848576302], [-163.75333248599708, 63.21944896102377], [-163.06722449445786, 63.05945872664802], [-162.26055538638175, 63.54193573674115], [-161.53444983624863, 63.455816962326764], [-160.7725066803211, 63.766108100023246], [-160.9583351308426, 64.22279857040274], [-161.51806840721218, 64.40278758407527], [-160.77777767641481, 64.78860382756642], [-161.39192623598765, 64.77723501246231], [-162.4530500966689, 64.55944468856819], [-162.75778601789415, 64.33860545516876], [-163.54639421288428, 64.5591604681905], [-164.96082984114514, 64.44694509546883], [-166.42528825586447, 64.68667206487066], [-166.8450042389391, 65.08889557561452], [-168.11056006576715, 65.66999705673675], [-166.70527116602193, 66.08831777613938], [-164.47470964257548, 66.5766600612975], [-163.65251176659564, 66.5766600612975], [-163.78860165103623, 66.07720734319668], [-161.67777442121013, 66.11611969671242], [-162.48971452538004, 66.73556509059512], [-163.71971696679117, 67.11639455837008], [-164.4309913808565, 67.61633820257777], [-165.39028683170673, 68.04277212185025], [-166.76444068099605, 68.35887685817966], [-166.20470740462667, 68.88303091091615], [-164.43081051334346, 68.91553538682774], [-163.1686136546145, 69.37111481391287], [-162.930566169262, 69.85806183539927], [-161.90889726463556, 70.33332998318764], [-160.93479651593367, 70.44768992784958], [-159.03917578838713, 70.89164215766891], [-158.11972286683394, 70.82472117785102], [-156.58082455139808, 71.35776357694175], [-155.06779029032427, 71.14777639432367], [-154.3441652089412, 70.69640859647018], [-153.9000062733926, 70.88998851183567], [-152.21000606993528, 70.82999217394485], [-152.27000240782613, 70.60000621202983], [-150.73999243874448, 70.43001658800569], [-149.7200030181675, 70.53001048449045], [-147.61336157935705, 70.2140349392418], [-145.68998980022533, 70.12000967068673], [-144.9200109590764, 69.98999176704046], [-143.58944618042523, 70.15251414659832], [-142.07251034871348, 69.85193817817265], [-140.98598752156073, 69.71199839952635], [-140.98598761037601, 69.71199839952635]]], [[[-171.73165686753944, 63.782515367275934], [-171.1144335602453, 63.59219106714495], [-170.4911124339407, 63.694975490973505], [-169.6825054596536, 63.43111562769119], [-168.6894394603007, 63.297506212000556], [-168.77194088445466, 63.18859813094544], [-169.5294398672051, 62.97693146427792], [-170.29055620021595, 63.194437567794424], [-170.67138566799093, 63.3758218451389], [-171.55306311753873, 63.317789211675105], [-171.79111060289122, 63.40584585230046], [-171.73165686753944, 63.782515367275934]]]]}