Skip to content

Commit

Permalink
Make JSON fixture compliant with Prettier
Browse files Browse the repository at this point in the history
  • Loading branch information
@chosak
chosak committed Sep 12, 2024
1 parent 4f95390 commit b1ebbdd
Show file tree
Hide file tree
Showing 3 changed files with 82 additions and 97 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ jobs:
- name: Check formatting
run: black . --check
- name: The test fixture should be kept in sync with the sample database
run: ./manage.py dumpdata --indent=4 crawler | diff crawler/fixtures/sample.json -
run: ./manage.py dumpdata --indent=2 crawler | diff crawler/fixtures/sample.json -
- name: Run Python tests
run: pytest

2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,7 @@ The Python tests make use of a test fixture generated from
To recreate this test fixture:

```
./manage.py dumpdata --indent=4 crawler > crawler/fixtures/sample.json
./manage.py dumpdata --indent=2 crawler > crawler/fixtures/sample.json
```

### Code formatting
Expand Down
175 changes: 80 additions & 95 deletions crawler/fixtures/sample.json
View file
Original file line number Diff line number Diff line change
@@ -1,157 +1,142 @@
[
{
{
"model": "crawler.crawl",
"pk": 1,
"fields": {
"started": "2024-09-11T16:41:20.036Z",
"status": "Finished",
"config": {
"start_url": "http://localhost:8000",
"max_pages": 0,
"depth": 0
},
"failure_message": null
"started": "2024-09-11T16:41:20.036Z",
"status": "Finished",
"config": {
"start_url": "http://localhost:8000",
"max_pages": 0,
"depth": 0
},
"failure_message": null
}
},
{
},
{
"model": "crawler.component",
"pk": 1,
"fields": {
"class_name": "o-sample"
"class_name": "o-sample"
}
},
{
},
{
"model": "crawler.link",
"pk": 1,
"fields": {
"href": "./file.xlsx"
"href": "./file.xlsx"
}
},
{
},
{
"model": "crawler.link",
"pk": 2,
"fields": {
"href": "/child/"
"href": "/child/"
}
},
{
},
{
"model": "crawler.link",
"pk": 3,
"fields": {
"href": "/child/?foo=bar"
"href": "/child/?foo=bar"
}
},
{
},
{
"model": "crawler.link",
"pk": 4,
"fields": {
"href": "/child/?page=2"
"href": "/child/?page=2"
}
},
{
},
{
"model": "crawler.link",
"pk": 5,
"fields": {
"href": "/child/?page=2&foo=bar"
"href": "/child/?page=2&foo=bar"
}
},
{
},
{
"model": "crawler.link",
"pk": 6,
"fields": {
"href": "https://example.com/"
"href": "https://example.com/"
}
},
{
},
{
"model": "crawler.link",
"pk": 7,
"fields": {
"href": "https://example.com/file.xlsx"
"href": "https://example.com/file.xlsx"
}
},
{
},
{
"model": "crawler.link",
"pk": 8,
"fields": {
"href": "https://example.org/"
"href": "https://example.org/"
}
},
{
},
{
"model": "crawler.link",
"pk": 9,
"fields": {
"href": "/"
"href": "/"
}
},
{
},
{
"model": "crawler.page",
"pk": 1,
"fields": {
"crawl": 1,
"timestamp": "2024-09-11T16:41:20.227Z",
"url": "http://localhost:8000/",
"title": "Sample homepage",
"language": "en",
"html": "<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <title>Sample homepage</title>\n <meta charset=\"utf-8\" />\n <meta http-equiv=\"Content-type\" content=\"text/html; charset=utf-8\" />\n <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\" />\n </head>\n <body>\n <h1>Sample homepage</h1>\n <p>This is sample content.</p>\n <div class=\"o-sample\">This is a sample component.</div>\n <p><a href=\"/child/\">This is a link to a child page.</a></p>\n <p><a href=\"https://example.com/\">This is a link somewhere else.</a></p>\n <p><a href=\"/external-site/?ext_url=https%3A%2F%2Fexample.org%2F\" data-pretty-href=\"https://example.org/\">This is an obfuscated link somewhere else.</a></p>\n <p><a href=\"/external-site/?ext_url=https%3A%2F%2Fexample.org%2F\" data-pretty-href=\"https://example.org/\">This is another obfuscated link some\n where else.</a></p>\n <p><a href=\"./file.xlsx\">This links to a file.</a></p>\n <p><a href=\"https://example.com/file.xlsx\">This links to a file somewhere else.</a></p>\n <p><a href=\"/child/?page=2\">This link has a page query string parameter.</a></p> <p><a href=\"/child/?foo=bar\">This link has a non-page query string parameter.</a></p>\n <p><a href=\"/child/?page=2&foo=bar\">This link has multiple query string parameters.</a></p>\n </body>\n</html>\n",
"text": "Sample homepage This is sample content. This is a sample component. This is a link to a child page. This is a link somewhere else. This is an obfuscated link somewhere else. This is another obfuscated link some where else. This links to a file. This links to a file somewhere else. This link has a page query string parameter. This link has a non-page query string parameter. This link has multiple query string parameters.",
"components": [
1
],
"links": [
1,
2,
3,
4,
5,
6,
7,
8
]
"crawl": 1,
"timestamp": "2024-09-11T16:41:20.227Z",
"url": "http://localhost:8000/",
"title": "Sample homepage",
"language": "en",
"html": "<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <title>Sample homepage</title>\n <meta charset=\"utf-8\" />\n <meta http-equiv=\"Content-type\" content=\"text/html; charset=utf-8\" />\n <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\" />\n </head>\n <body>\n <h1>Sample homepage</h1>\n <p>This is sample content.</p>\n <div class=\"o-sample\">This is a sample component.</div>\n <p><a href=\"/child/\">This is a link to a child page.</a></p>\n <p><a href=\"https://example.com/\">This is a link somewhere else.</a></p>\n <p><a href=\"/external-site/?ext_url=https%3A%2F%2Fexample.org%2F\" data-pretty-href=\"https://example.org/\">This is an obfuscated link somewhere else.</a></p>\n <p><a href=\"/external-site/?ext_url=https%3A%2F%2Fexample.org%2F\" data-pretty-href=\"https://example.org/\">This is another obfuscated link some\n where else.</a></p>\n <p><a href=\"./file.xlsx\">This links to a file.</a></p>\n <p><a href=\"https://example.com/file.xlsx\">This links to a file somewhere else.</a></p>\n <p><a href=\"/child/?page=2\">This link has a page query string parameter.</a></p> <p><a href=\"/child/?foo=bar\">This link has a non-page query string parameter.</a></p>\n <p><a href=\"/child/?page=2&foo=bar\">This link has multiple query string parameters.</a></p>\n </body>\n</html>\n",
"text": "Sample homepage This is sample content. This is a sample component. This is a link to a child page. This is a link somewhere else. This is an obfuscated link somewhere else. This is another obfuscated link some where else. This links to a file. This links to a file somewhere else. This link has a page query string parameter. This link has a non-page query string parameter. This link has multiple query string parameters.",
"components": [1],
"links": [1, 2, 3, 4, 5, 6, 7, 8]
}
},
{
},
{
"model": "crawler.page",
"pk": 2,
"fields": {
"crawl": 1,
"timestamp": "2024-09-11T16:41:20.679Z",
"url": "http://localhost:8000/child/?page=2",
"title": "Sample child page",
"language": "en",
"html": "<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <title>Sample child page</title>\n <meta charset=\"utf-8\" />\n <meta http-equiv=\"Content-type\" content=\"text/html; charset=utf-8\" />\n <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\" />\n </head>\n <body>\n <h1>Sample child page</h1>\n <p>This is sample content.</p>\n <p><a href=\"/\">This is a link to the homepage.</a></p>\n </body>\n</html>\n",
"text": "Sample child page This is sample content. This is a link to the homepage.",
"components": [],
"links": [
9
]
"crawl": 1,
"timestamp": "2024-09-11T16:41:20.679Z",
"url": "http://localhost:8000/child/?page=2",
"title": "Sample child page",
"language": "en",
"html": "<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <title>Sample child page</title>\n <meta charset=\"utf-8\" />\n <meta http-equiv=\"Content-type\" content=\"text/html; charset=utf-8\" />\n <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\" />\n </head>\n <body>\n <h1>Sample child page</h1>\n <p>This is sample content.</p>\n <p><a href=\"/\">This is a link to the homepage.</a></p>\n </body>\n</html>\n",
"text": "Sample child page This is sample content. This is a link to the homepage.",
"components": [],
"links": [9]
}
},
{
},
{
"model": "crawler.page",
"pk": 3,
"fields": {
"crawl": 1,
"timestamp": "2024-09-11T16:41:23.003Z",
"url": "http://localhost:8000/child/",
"title": "Sample child page",
"language": "en",
"html": "<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <title>Sample child page</title>\n <meta charset=\"utf-8\" />\n <meta http-equiv=\"Content-type\" content=\"text/html; charset=utf-8\" />\n <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\" />\n </head>\n <body>\n <h1>Sample child page</h1>\n <p>This is sample content.</p>\n <p><a href=\"/\">This is a link to the homepage.</a></p>\n </body>\n</html>\n",
"text": "Sample child page This is sample content. This is a link to the homepage.",
"components": [],
"links": [
9
]
"crawl": 1,
"timestamp": "2024-09-11T16:41:23.003Z",
"url": "http://localhost:8000/child/",
"title": "Sample child page",
"language": "en",
"html": "<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <title>Sample child page</title>\n <meta charset=\"utf-8\" />\n <meta http-equiv=\"Content-type\" content=\"text/html; charset=utf-8\" />\n <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\" />\n </head>\n <body>\n <h1>Sample child page</h1>\n <p>This is sample content.</p>\n <p><a href=\"/\">This is a link to the homepage.</a></p>\n </body>\n</html>\n",
"text": "Sample child page This is sample content. This is a link to the homepage.",
"components": [],
"links": [9]
}
},
{
},
{
"model": "crawler.error",
"pk": 1,
"fields": {
"crawl": 1,
"timestamp": "2024-09-11T16:41:22.353Z",
"url": "https://example.com/file.xlsx",
"status_code": 404,
"referrer": "http://localhost:8000/"
"crawl": 1,
"timestamp": "2024-09-11T16:41:22.353Z",
"url": "https://example.com/file.xlsx",
"status_code": 404,
"referrer": "http://localhost:8000/"
}
}
}
]

0 comments on commit b1ebbdd

Please sign in to comment.