Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

WIP - New feed for importer #79

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

WIP - New feed for importer #79

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions .env
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,10 @@ POSTGRES_PASSWORD=DatabaseFTW
POSTGRES_USER=postgres
REGISTRY_HOST=ghcr.io
REGISTRY_URI=/scientist-softserv/arce
SITE_URI=web.oh.staging.notch8network.com
SITE_URI=https://arce-staging.notch8.cloud
SOLR_URL=http://solr:8983/solr/blacklight-core
TAG=dev
TEST_DB=arce_test
IIIF_URL=http://web.arce.staging.notch8network.com
IIIF_URL=http://web.arce.staging.notch8network.com
ROOT_URL=http://arce.test # need this for the URLs for related items. Will need to add to staging and prod envs
```
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
/public/packs-test
/node_modules
public/packs/*
public/assets/*
.DS_Store
.env.*
.env.*
Expand Down
2 changes: 1 addition & 1 deletion app/controllers/catalog_controller.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,9 +133,9 @@ class CatalogController < ApplicationController
config.add_show_field 'copyright_status_t', label: 'Copyright Status', highlight: true
config.add_show_field 'note_license_t', label: 'Creative Commons License', highlight: true, helper_method: 'external_link'
config.add_show_field 'note_rights_t', label: 'Rights Statement', highlight: true
config.add_show_field 'file_name_t', label: 'Local ID', highlight: true
config.add_show_field 'project_history_t', label: 'Project History'
config.add_show_field 'note_funding_t', label: 'Funding Agency'
config.add_show_field 'related_items_t', label: 'Related Items'
# config.add_show_field 'resource_preview_t', label: 'Image preview url'
config.add_show_field 'resource_url_t', label: 'Resource url'

Expand Down
75 changes: 40 additions & 35 deletions app/models/arce_item.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,22 +24,22 @@ def self.index_logger
end

def self.client(args)
url = args[:url] || "https://dl.library.ucla.edu/oai2/"
url = args[:url] || "https://digital.library.ucla.edu/catalog/oai/"
OAI::Client.new url,
:headers => { "From" => "[email protected]" },
:parser => 'rexml',
metadata_prefix: 'mods',
metadata_prefix: 'mods_arce',
verb: 'ListRecords'
end

def self.fetch(args)
set = args[:set] || "arce_1"
response = client(args).list_records(set: set, metadata_prefix: 'mods')
set = args[:set] || "oai_set_ssim:arce"
response = client(args).list_records(set: set, metadata_prefix: 'mods_arce')
response
end

def self.get(args)
response = client(args).get_record(identifier: args[:identifier], metadata_prefix: 'mods')
response = client(args).get_record(identifier: args[:identifier], metadata_prefix: 'mods_arce')
response
end

Expand Down Expand Up @@ -125,9 +125,11 @@ def self.process_record(record)
end

# rubocop:disable Metrics/LineLength
history = ArceItem.find_or_new(record.header.identifier.split(':').last) # Digest::MD5.hexdigest(record.header.identifier).to_i(16))
record_id = record.header.identifier.split(':').last[1..-1].gsub('/', '-')
history = ArceItem.find_or_new(record_id)
# rubocop:enable Metrics/LineLength
history.attributes['id_t'] = record.header.identifier.split(':').last
history.attributes['id_t'] = record_id

if record.header.datestamp
history.attributes[:timestamp] = Time.parse(record.header.datestamp)
end
Expand All @@ -137,42 +139,20 @@ def self.process_record(record)

batch.children.each do |child|
next if child.class == REXML::Text

if child.attributes['displayLabel'] == 'File name'
file_name = child.text
history.attributes['file_name_t'] ||= []
if !history.attributes['file_name_t'].include?(file_name)
history.attributes['file_name_t'] << file_name
end
end
if child.name == 'identifier'
if child.attributes['type'] == 'local'
if child.attributes['displayLabel'] == "Local ID"
child.children.each do |ch|
next if ch.class == REXML::Text

file_name = ch.text
history.attributes['file_name_t'] ||= []
if !history.attributes['file_name_t'].include?(file_name)
history.attributes['file_name_t'] << file_name
end
end
end
end
end
if child.name == 'relatedItem'
if child.attributes['type'] == 'host'
child.children.each do |ch|
next if ch.class == REXML::Text

ch.children.each do |c|
next if c.class == REXML::Text

collections_to_skip = ["Africa Collections", "Middle East & North Africa Collections"]
if c.name == 'title'
history.attributes['collection_display'] ||= c.text
history.attributes['collection_facet'] ||= c.text
history.attributes['collection_sort'] ||= c.text
history.attributes['collection_t'] ||= c.text
byebug
history.attributes['collection_display'] = c.text unless collections_to_skip.include?(c.text)
history.attributes['collection_facet'] = c.text unless collections_to_skip.include?(c.text)
history.attributes['collection_sort'] = c.text unless collections_to_skip.include?(c.text)
history.attributes['collection_t'] = c.text unless collections_to_skip.include?(c.text)
end
end
end
Expand All @@ -191,6 +171,31 @@ def self.process_record(record)
end
end
end

if child.attributes['otherType'] == 'relatedTo'
child.children.each do |ch|
next if ch.class == REXML::Text

ch.children.each do |c|
next if c.class == REXML::Text
history.attributes['related_items_display'] ||= []
history.attributes['related_items_t'] ||= []
if c.name == 'title'
related_items = c.text
related_items.gsub("digital.library.ucla.edu/catalog/ark:", "archives.arce.org/catalog/")
# https://digital.library.ucla.edu/catalog/ark:/21198/z1bc8zr8
# http://arce.test/catalog/%2F21198%2Fz1kh5nht
if !history.attributes['related_items_t'].include?(related_items)
history.attributes['related_items_t'] << related_items
end
if !history.attributes['related_items_display'].include?(related_items)
history.attributes['related_items_display'] << related_items
end
end
end
end
end

if child.attributes['type'] == 'series'
child.children.each do |ch|
next if ch.class == REXML::Text
Expand Down
2 changes: 1 addition & 1 deletion app/views/catalog/_uv.html.erb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<div class="container text-center g-mb-20">
<iframe
class="universal-viewer-iframe"
src="<%= request&.base_url %>/uv/uv.html#?manifest=<%= riiif_manifest_url(id: @document.id) %>"
src="<%= request&.base_url %>/uv/uv.html#?manifest=<%= @document['resource_url_t'].first %> "
width="640"
height="480"
allowfullscreen
Expand Down