forked from NMAAHC/nmaahcmm
-
Notifications
You must be signed in to change notification settings - Fork 0
/
makemetadata
executable file
·203 lines (195 loc) · 12.5 KB
/
makemetadata
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
#!/usr/bin/env bash
# a script to generate metadata sidecar files for video files
# load nmaahcmmfunctions into this script
SCRIPT_PATH="${0%/*}"
. "${SCRIPT_PATH}/nmaahcmmfunctions"
[[ -f "${SCRIPT_PATH}/nmaahcmmfunctions" ]] || { echo "Missing '${SCRIPT_PATH}/nmaahcmmfunctions'. Exiting." ; exit 1 ;};
_initialize_make # safe script termination process defined in nmaahcmmfunctions
DEPENDENCIES=(tree ffprobe mediaconch exiftool qcli md5deep mkvpropedit) # list dependencies required by script
_check_dependencies "${DEPENDENCIES[@]}" # defined in nmaahcmmfunctions
## USAGE
_usage(){
echo
echo "$(basename "${0}")"
echo
echo "This application will generate metadata sidecar files for all video files in a package. It takes one or more files or packages as input."
echo "The application generates all of the following files: a tree XML, a MediaInfo XML, a MediaTrace XML, an FFprobe XML, and an Exiftool XML."
echo "The option -m will generate an MD5 sidecar file. The option -q will generate a QCTools XML as well; this option is recommended for digitized video only. Both of these files may take a while to generate, depending on the size of your video file."
echo "The application will check for existing sidecar files before creating them again. To overwrite any existing metadata files, use option -o."
echo "If the video file is an MKV file, the script can embed a copy of the metadata files it generates into the MKV wrapper using option -e."
echo
echo "Usage: $(basename ${0}) [ -e | -m | -o | -q | -h ] fileorpackage1 [ fileorpackage2 ...]"
echo " -e embed metadata files in MKV files"
echo " -m generate MD5 file"
echo " -o overwrite existing metadata files"
echo " -q generate QCTools XML"
echo " -h (display this help menu)"
exit
}
[ "${#}" = 0 ] && _usage # if the command is run with no arguments then _usage is called
# getopts loop
OPTIND=1
while getopts ":hemoq" OPT; do
case "${OPT}" in
h) _usage ;; # if the operator runs "[scriptname] -h" then the _usage text above will display in the terminal
e) EMBED="Y" ;;
m) MD5="Y" ;;
o) OVERWRITE="Y" ;;
q) QCTOOLS="Y" ;;
*) echo "Invalid option -${OPTARG}" ; _usage ;; # if the operator tries to use an option other than the ones listed above, the _usage text will display in the terminal
esac
done
shift $(( ${OPTIND} - 1 ))
## SCRIPT ACTIONS
# log script beginning
_log -b
while [ "${*}" != "" ] ; do
# process if INPUT is a file
if [[ -f "${1}" ]] ; then
FILE="${1}"
_report -gs "Making metadata reports for ${FILE}..."
BASENAME=$(basename "${FILE}")
OUTPUT_DIR="$(dirname "${FILE}")/metadata/"
mkdir -p "${OUTPUT_DIR}"
TREE_OUT="${OUTPUT_DIR}/${BASENAME}_tree.xml"
FFPROBE_OUT="${OUTPUT_DIR}/${BASENAME}_ffprobe.xml"
MEDIAINFO_OUT="${OUTPUT_DIR}/${BASENAME}_mediainfo.xml"
MEDIATRACE_OUT="${OUTPUT_DIR}/${BASENAME}_mediatrace.xml"
EXIFTOOL_OUT="${OUTPUT_DIR}/${BASENAME}_exiftool.xml"
# if requested: remove existing metadata files to be overwritten
if [[ "${OVERWRITE}" == "Y" ]] ; then
rm "${FFPROBE_OUT}" "${MEDIAINFO_OUT}" "${MEDIATRACE_OUT}" "${EXIFTOOL_OUT}"
fi
# if the metadata files don't already exist, create them; the command following "&&" will only execute if the test before it is successful
[[ ! -f "${TREE_OUT}" ]] && tree -DaNXs --du --timefmt "%Y-%m-%dT%H:%M:%SZ" "$(dirname "${FILE}")" > "${TREE_OUT}"
[[ ! -f "${FFPROBE_OUT}" ]] && ffprobe 2> /dev/null "${FILE}" -show_format -show_streams -show_data -show_error -show_versions -show_chapters -noprivate -of xml="q=1:x=1" > "${FFPROBE_OUT}"
[[ ! -f "${MEDIAINFO_OUT}" ]] && mediaconch -mi -fx "${FILE}" > "${MEDIAINFO_OUT}"
[[ ! -f "${MEDIATRACE_OUT}" ]] && mediaconch -mt -fx "${FILE}" | xml fo > "${MEDIATRACE_OUT}"
[[ ! -f "${EXIFTOOL_OUT}" ]] && exiftool -X "${FILE}" > "${EXIFTOOL_OUT}"
# generate QCTools XML if requested; overwrite if requested
if [[ "${QCTOOLS}" == "Y" ]] ; then
QCTOOLS_OUT="${OUTPUT_DIR}/${BASENAME}.qctools.xml.gz"
if [[ "${OVERWRITE}" == "Y" ]] ; then
rm "${QCTOOLS_OUT}"
qcli -i "${FILE}" -o "${QCTOOLS_OUT}"
elif [[ ! -f "${QCTOOLS_OUT}" ]] ; then
qcli -i "${FILE}" -o "${QCTOOLS_OUT}"
fi
fi
# embed metadata files in MKV file if requested
# to see what attachments are in your file and what ID #s they have, run: mkvmerge --identify ${filename}
# to extract those attachments, run: mkvextract ${filename}.mkv attachments ${ID#}:${extracted_output_filename.ext} ---> e.g. if you see that attachment 1 is a mediainfo xml and you want to extract it into a file on the Desktop called "mediainfo.xml," run: mkvextract file.mkv 1:~/Desktop/mediainfo.xml
# to delete attachments, run: mkvpropedit ${filename}.mkv --delete-attachment ${ID#}
# right now, the script will just add more and more attachments if it's run on the same file multiple times. MKV attachments are identified by numbers, so it can be complicated to delete/overwrite based on content.
if [[ "${EMBED}" == "Y" ]] ; then
if [[ "${FILE}" == *.mkv ]] || [[ "${FILE}" == *.MKV ]] ; then
_report -gs "Embedding metadata reports in MKV file ${FILE}..."
mkvpropedit "${FILE}" --attachment-description "FFprobe report" --add-attachment "${FFPROBE_OUT}"
mkvpropedit "${FILE}" --attachment-description "MediaInfo report" --add-attachment "${MEDIAINFO_OUT}"
mkvpropedit "${FILE}" --attachment-description "MediaTrace report" --add-attachment "${MEDIATRACE_OUT}"
mkvpropedit "${FILE}" --attachment-description "ExifTool report" --add-attachment "${EXIFTOOL_OUT}"
if [[ "${QCTOOLS}" = "Y" ]] ; then
mkvpropedit "${FILE}" --attachment-description "QCTools report from vrecord capture process (zipped XML)" --add-attachment "${QCTOOLS_OUT}"
fi
fi
fi
# generate MD5 checksum if requested; overwrite if requested
if [[ "${MD5}" == "Y" ]] ; then
MD5FILE="${OUTPUT_DIR}/${BASENAME}.md5"
if [[ "${OVERWRITE}" == "Y" ]] ; then
rm "${MD5FILE}"
md5deep -bre "${FILE}" >> "${MD5FILE}" # -b=strip leading directory info, -r=recursive, -e=display progress indicator
elif [[ ! -f "${MD5FILE}" ]] ; then
md5deep -bre "${FILE}" >> "${MD5FILE}"
fi
fi
_report "Moving to next file if present..."
elif [[ -d "${1}" ]] ; then
# process if INPUT is a directory/package
TEMPFILE1="$(_maketemp)"
TEMPFILE2="$(_maketemp)"
INPUT="${1}"
# Gather a list of potential video files in the directory/package (TEMPFILE1). Ignore common image, audio, and text files; ignore all files in a "metadata" folder.
_report "Checking for video files in the package..."
find "${INPUT}" -type f -not -iname "*.dpx" -not -iname "*.wav" -not -iname "*.mp3" -not -iname "*.txt" -not -iname "*.xml" -not -path "*/metadata/*" -not -path "*/ACCESS/*" >> "${TEMPFILE1}"
# Check whether there is a video stream in each file; if there is, add it to a list of confirmed video files (TEMPFILE2). This approach is intended to catch all video files regardless of type or extension.
while read FILE ; do
if [[ -n "$(mediainfo --Inform="General;%VideoCount%" "${FILE}")" ]] ; then
echo "Found video file ${FILE}"
echo "${FILE}" >> "${TEMPFILE2}"
else
:
fi
done <"${TEMPFILE1}"
# check whether any video files were found in this process, by checking whether $TEMPFILE2 is empty.
if [[ ! -s "${TEMPFILE2}" ]] ; then
_report -r "There are no video files in this package!"
fi
# create metadata files
while read FILE ; do
_report -gs "Making metadata reports for ${FILE}..."
BASENAME=$(basename "${FILE}")
OUTPUT_DIR="${INPUT}/metadata/"
mkdir -p "${OUTPUT_DIR}"
TREE_OUT="${OUTPUT_DIR}/${BASENAME}_tree.txt"
FFPROBE_OUT="${OUTPUT_DIR}/${BASENAME}_ffprobe.xml"
MEDIAINFO_OUT="${OUTPUT_DIR}/${BASENAME}_mediainfo.txt"
MEDIATRACE_OUT="${OUTPUT_DIR}/${BASENAME}_mediatrace.xml"
EXIFTOOL_OUT="${OUTPUT_DIR}/${BASENAME}_exiftool.csv"
# if requested: remove existing metadata files to be overwritten
if [[ "${OVERWRITE}" == "Y" ]] ; then
rm "${FFPROBE_OUT}" "${MEDIAINFO_OUT}" "${MEDIATRACE_OUT}" "${EXIFTOOL_OUT}"
fi
# check if the metadata files exist with size > 0; if not, create them. The command following "&&" will only execute if the test before it is successful.
[[ ! -s "${TREE_OUT}" ]] && echo "Command: tree -DaNs --du --timefmt \"%Y-%m-%dT%H:%M:%SZ\" ${INPUT}" >> "${TREE_OUT}" && tree -DaNs --du --timefmt "%Y-%m-%dT%H:%M:%SZ" "${INPUT}" >> "${TREE_OUT}"
[[ ! -s "${FFPROBE_OUT}" ]] && ffprobe 2> /dev/null "${FILE}" -show_format -show_streams -show_data -show_error -show_versions -show_chapters -noprivate -of xml="q=1:x=1" > "${FFPROBE_OUT}"
[[ ! -s "${MEDIAINFO_OUT}" ]] && mediainfo -f "${FILE}" > "${MEDIAINFO_OUT}"
[[ ! -s "${MEDIATRACE_OUT}" ]] && mediaconch -mt -fx "${FILE}" | xml fo > "${MEDIATRACE_OUT}"
[[ ! -s "${EXIFTOOL_OUT}" ]] && exiftool -csv "${FILE}" > "${EXIFTOOL_OUT}"
# generate QCTools XML if requested; overwrite if requested
if [[ "${QCTOOLS}" == "Y" ]] ; then
QCTOOLS_OUT="${OUTPUT_DIR}/${BASENAME}.qctools.xml.gz"
if [[ "${OVERWRITE}" == "Y" ]] ; then
rm "${QCTOOLS_OUT}"
qcli -i "${FILE}" -o "${QCTOOLS_OUT}"
elif [[ ! -f "${QCTOOLS_OUT}" ]] ; then
qcli -i "${FILE}" -o "${QCTOOLS_OUT}"
fi
fi
# embed metadata files in MKV file if requested
# to see what attachments are in your file and what ID #s they have, run: mkvmerge --identify ${filename}
# to extract those attachments, run: mkvextract ${filename}.mkv attachments ${ID#}:${extracted_output_filename.ext} ---> e.g. if you see that attachment 1 is a mediainfo xml and you want to extract it into a file on the Desktop called "mediainfo.xml," run: mkvextract file.mkv 1:~/Desktop/mediainfo.xml
# to delete attachments, run: mkvpropedit ${filename}.mkv --delete-attachment ${ID#}
# right now, the script will just add more and more attachments if it's run on the same file multiple times. MKV attachments are identified by numbers, so it can be complicated to delete/overwrite based on content.
if [[ "${EMBED}" == "Y" ]] ; then
if [[ "${FILE}" == *.mkv ]] || [[ "${FILE}" == *.MKV ]] ; then
_report -gs "Embedding metadata reports in MKV file ${FILE}..."
mkvpropedit "${FILE}" --attachment-description "FFprobe report" --add-attachment "${FFPROBE_OUT}"
mkvpropedit "${FILE}" --attachment-description "MediaInfo report" --add-attachment "${MEDIAINFO_OUT}"
mkvpropedit "${FILE}" --attachment-description "MediaTrace report" --add-attachment "${MEDIATRACE_OUT}"
mkvpropedit "${FILE}" --attachment-description "ExifTool report" --add-attachment "${EXIFTOOL_OUT}"
if [[ "${QCTOOLS}" = "Y" ]] ; then
mkvpropedit "${FILE}" --attachment-description "QCTools report from vrecord capture process (zipped XML)" --add-attachment "${QCTOOLS_OUT}"
fi
fi
fi
# generate MD5 checksum if requested; overwrite if requested
if [[ "${MD5}" == "Y" ]] ; then
MD5FILE="${OUTPUT_DIR}/${BASENAME}.md5"
if [[ "${OVERWRITE}" == "Y" ]] ; then
rm "${MD5FILE}"
md5deep -bre "${FILE}" >> "${MD5FILE}" # -b=strip leading directory info, -r=recursive, -e=display progress indicator
elif [[ ! -f "${MD5FILE}" ]] ; then
md5deep -bre "${FILE}" >> "${MD5FILE}"
fi
fi
_report "Moving to next file if present..."
done <"${TEMPFILE2}"
fi
shift
done
# log script ending
rm "${TEMPFILE1}" "${TEMPFILE2}"
_log -e
_report -g "makemetadata process complete."
echo