Filter Python CD index results with queries

This is more efficient than a single DELETE statement.

examples/cdindex/cd5index-all-py.sql

@@ -0,0 +1,6 @@
+-- Export the CD5 index for works where one can be calculated
+
+CREATE INDEX IF NOT EXISTS rolap.valid_cd5index_doi_idx ON valid_cd5index(doi);
+
+SELECT doi, cdindex FROM rolap.cdindex
+  INNER JOIN rolap.valid_cd5index USING(doi);;

examples/cdindex/cdindex-db.py

@@ -152,18 +152,4 @@ def process_batch(start):
 
     db.commit()
 
-    db.execute("""
-    -- Works and references
-    ATTACH 'cdindex.db' AS wr;
-
-    DELETE FROM cdindex
-    WHERE doi NOT IN (
-      SELECT cdindex.doi FROM cdindex
-          INNER JOIN wr.works USING(doi)
-          WHERE works.published_year <= 2018 OR
-          (SELECT 1 FROM work_references WHERE work_id == works.id)
-    );
-      """)
-    perf.log("Remove invalid records")
-
     db.close()

examples/cdindex/valid_cd5index.sql

@@ -0,0 +1,9 @@
+-- Create a table of publications for which a valid CD5 index can be calculated
+
+CREATE INDEX IF NOT EXISTS rolap.cdindex_doi_idx ON cdindex(doi);
+
+CREATE TABLE rolap.valid_cd5index AS
+  SELECT cdindex.doi FROM rolap.cdindex
+      INNER JOIN works USING(doi)
+      WHERE works.published_year <= 2018 AND
+      (SELECT 1 FROM work_references WHERE work_id == works.id);