From 69a123bc997802bce50c4fd125690fa78b50f1ea Mon Sep 17 00:00:00 2001 From: dell Date: Sat, 17 Aug 2024 21:27:46 +0800 Subject: [PATCH] mathjsx --- _config.yml | 34 +-- _includes/analytics/cloudflare.html | 7 + _includes/analytics/goatcounter.html | 6 + _includes/analytics/google.html | 13 + _includes/analytics/matomo.html | 14 + _includes/analytics/umami.html | 6 + _includes/comments.html | 5 + _includes/comments/disqus.html | 50 ++++ _includes/comments/giscus.html | 71 +++++ _includes/comments/utterances.html | 49 ++++ _includes/datetime.html | 20 ++ _includes/embed/audio.html | 35 +++ _includes/embed/bilibili.html | 9 + _includes/embed/twitch.html | 8 + _includes/embed/video.html | 59 ++++ _includes/embed/youtube.html | 9 + _includes/favicons.html | 19 ++ _includes/footer.html | 49 ++++ _includes/head.html | 107 ++++++++ _includes/js-selector.html | 106 ++++++++ _includes/jsdelivr-combine.html | 26 ++ _includes/lang.html | 10 + _includes/language-alias.html | 70 +++++ _includes/mathjax_support.html | 30 +++ _includes/media-url.html | 37 +++ _includes/mermaid.html | 62 +++++ _includes/metadata-hook.html | 1 + _includes/mode-toggle.html | 116 ++++++++ _includes/no-linenos.html | 10 + _includes/notification.html | 24 ++ _includes/origin-type.html | 13 + _includes/pageviews/goatcounter.html | 19 ++ _includes/post-description.html | 16 ++ _includes/post-nav.html | 34 +++ _includes/post-paginator.html | 91 +++++++ _includes/post-sharing.html | 52 ++++ _includes/read-time.html | 37 +++ _includes/refactor-content.html | 255 ++++++++++++++++++ _includes/related-posts.html | 94 +++++++ _includes/search-loader.html | 47 ++++ _includes/search-results.html | 10 + _includes/sidebar.html | 99 +++++++ _includes/toc.html | 13 + _includes/topbar.html | 77 ++++++ _includes/trending-tags.html | 46 ++++ _includes/update-list.html | 40 +++ _layouts/archives.html | 35 +++ _layouts/categories.html | 138 ++++++++++ _layouts/category.html | 24 ++ _layouts/compress.html | 10 + _layouts/default.html | 82 ++++++ _layouts/home.html | 115 ++++++++ _layouts/page.html | 20 ++ _layouts/post.html | 152 +++++++++++ _layouts/tag.html | 23 ++ _layouts/tags.html | 22 ++ _posts/2024-08-11-first.md | 11 + ...73\347\232\204\345\261\236\346\200\247.md" | 98 +++++++ ...57\345\276\204\346\216\242\347\264\242.md" | 95 +++++++ ...72\344\270\216\345\244\215\345\216\237.md" | 43 +++ ...47\276\216\345\214\226\347\211\210json.md" | 34 +++ ...00\345\244\232\350\277\233\347\250\213.md" | 77 ++++++ ...04\347\237\245\350\257\206\346\261\240.md" | 62 +++++ ...345\256\232GPU\351\203\250\347\275\262.md" | 61 +++++ ...40\351\231\244\347\233\270\345\205\263.md" | 172 ++++++++++++ ...72\277\345\256\211\350\243\205anaconda.md" | 100 +++++++ ...76\346\216\245\347\233\270\345\205\263.md" | 69 +++++ ...0\254\350\231\253\345\233\276\350\247\243" | 110 ++++++++ ...db\347\232\204\345\205\263\351\227\255.md" | 65 +++++ ...50\344\274\240\346\226\207\344\273\266.md" | 47 ++++ ...53\347\247\221\345\255\246\345\256\266.md" | 32 +++ ...60\345\213\222\345\205\254\345\274\217.md" | 62 +++++ _posts/pool/2024-08-13-generator.md | 50 ++++ 73 files changed, 3797 insertions(+), 17 deletions(-) create mode 100644 _includes/analytics/cloudflare.html create mode 100644 _includes/analytics/goatcounter.html create mode 100644 _includes/analytics/google.html create mode 100644 _includes/analytics/matomo.html create mode 100644 _includes/analytics/umami.html create mode 100644 _includes/comments.html create mode 100644 _includes/comments/disqus.html create mode 100644 _includes/comments/giscus.html create mode 100644 _includes/comments/utterances.html create mode 100644 _includes/datetime.html create mode 100644 _includes/embed/audio.html create mode 100644 _includes/embed/bilibili.html create mode 100644 _includes/embed/twitch.html create mode 100644 _includes/embed/video.html create mode 100644 _includes/embed/youtube.html create mode 100644 _includes/favicons.html create mode 100644 _includes/footer.html create mode 100644 _includes/head.html create mode 100644 _includes/js-selector.html create mode 100644 _includes/jsdelivr-combine.html create mode 100644 _includes/lang.html create mode 100644 _includes/language-alias.html create mode 100644 _includes/mathjax_support.html create mode 100644 _includes/media-url.html create mode 100644 _includes/mermaid.html create mode 100644 _includes/metadata-hook.html create mode 100644 _includes/mode-toggle.html create mode 100644 _includes/no-linenos.html create mode 100644 _includes/notification.html create mode 100644 _includes/origin-type.html create mode 100644 _includes/pageviews/goatcounter.html create mode 100644 _includes/post-description.html create mode 100644 _includes/post-nav.html create mode 100644 _includes/post-paginator.html create mode 100644 _includes/post-sharing.html create mode 100644 _includes/read-time.html create mode 100644 _includes/refactor-content.html create mode 100644 _includes/related-posts.html create mode 100644 _includes/search-loader.html create mode 100644 _includes/search-results.html create mode 100644 _includes/sidebar.html create mode 100644 _includes/toc.html create mode 100644 _includes/topbar.html create mode 100644 _includes/trending-tags.html create mode 100644 _includes/update-list.html create mode 100644 _layouts/archives.html create mode 100644 _layouts/categories.html create mode 100644 _layouts/category.html create mode 100644 _layouts/compress.html create mode 100644 _layouts/default.html create mode 100644 _layouts/home.html create mode 100644 _layouts/page.html create mode 100644 _layouts/post.html create mode 100644 _layouts/tag.html create mode 100644 _layouts/tags.html create mode 100644 _posts/2024-08-11-first.md create mode 100644 "_posts/coding/2024-08-13-Python\350\216\267\345\217\226\347\261\273\347\232\204\345\261\236\346\200\247.md" create mode 100644 "_posts/coding/2024-08-13-chatglm-PPO\350\256\255\347\273\203\350\267\257\345\276\204\346\216\242\347\264\242.md" create mode 100644 "_posts/coding/2024-08-13-conda\347\216\257\345\242\203\345\272\223\344\277\241\346\201\257\345\257\274\345\207\272\344\270\216\345\244\215\345\216\237.md" create mode 100644 "_posts/coding/2024-08-13-json.dumps\350\276\223\345\207\272\347\276\216\345\214\226\347\211\210json.md" create mode 100644 "_posts/coding/2024-08-13-python\345\274\200\345\244\232\350\277\233\347\250\213.md" create mode 100644 "_posts/coding/2024-08-13-\346\225\260\346\215\256\347\273\223\346\236\204\347\237\245\350\257\206\346\261\240.md" create mode 100644 "_posts/coding/2024-08-13-\350\204\232\346\234\254\345\206\205\346\214\207\345\256\232GPU\351\203\250\347\275\262.md" create mode 100644 "_posts/engineering/2024-08-13-linux\345\210\240\351\231\244\347\233\270\345\205\263.md" create mode 100644 "_posts/engineering/2024-08-13-linux\346\234\215\345\212\241\345\231\250\347\246\273\347\272\277\345\256\211\350\243\205anaconda.md" create mode 100644 "_posts/engineering/2024-08-13-linux\350\275\257\351\223\276\346\216\245\347\233\270\345\205\263.md" create mode 100644 "_posts/engineering/2024-08-13-python\347\210\254\350\231\253\345\233\276\350\247\243" create mode 100644 "_posts/engineering/2024-08-13-transformers.Trainer\344\270\255wandb\347\232\204\345\205\263\351\227\255.md" create mode 100644 "_posts/engineering/2024-08-13-\344\270\244\345\217\260Linux\346\234\272\345\231\250\344\274\240\346\226\207\344\273\266.md" create mode 100644 "_posts/game/2024-06-27-\347\214\253\345\233\275\345\273\272\350\256\276\350\200\205\345\216\237\347\211\210\345\274\200\345\220\257\345\260\217\347\214\253\347\247\221\345\255\246\345\256\266.md" create mode 100644 "_posts/math/2024-08-13-\346\263\260\345\213\222\345\205\254\345\274\217.md" create mode 100644 _posts/pool/2024-08-13-generator.md diff --git a/_config.yml b/_config.yml index 2e76174..bc032f4 100644 --- a/_config.yml +++ b/_config.yml @@ -9,24 +9,24 @@ theme: jekyll-theme-chirpy lang: en # Change to your timezone › https://kevinnovak.github.io/Time-Zone-Picker -timezone: +timezone: Asia/Shanghai # jekyll-seo-tag settings › https://github.com/jekyll/jekyll-seo-tag/blob/master/docs/usage.md # ↓ -------------------------- -title: Chirpy # the main title +title: XYZ # the main title -tagline: A text-focused Jekyll theme # it will display as the sub-title +tagline: CMX,DYB,TWZ ---XYZ # it will display as the sub-title description: >- # used by seo meta and the atom feed - A minimal, responsive and feature-rich Jekyll theme for technical writing. + 快乐一家人 # Fill in the protocol & hostname for your site. # e.g. 'https://username.github.io', note that it does not end with a '/'. -url: "" +url: "https://zdongs.github.io" github: - username: github_username # change to your github username + username: zdongs # change to your github username twitter: username: twitter_username # change to your twitter username @@ -34,15 +34,16 @@ twitter: social: # Change to your full name. # It will be displayed as the default author of the posts and the copyright owner in the Footer - name: your_full_name + name: XYZ email: example@domain.com # change to your email address links: # The first element serves as the copyright owner's link - https://twitter.com/username # change to your twitter homepage - - https://github.com/username # change to your github homepage + - https://github.com/zdongs # change to your github homepage # Uncomment below to add more social links # - https://www.facebook.com/username # - https://www.linkedin.com/in/username + # - https://www.linkedin.com/ # Site Verification Settings webmaster_verifications: @@ -93,10 +94,10 @@ theme_mode: # [light | dark] # will be added to all media resources (site avatar, posts' images, audio and video files) paths starting with '/' # # e.g. 'https://cdn.com' -cdn: +cdn: "https://chirpy-img.netlify.app" # the avatar on sidebar, support local or CORS resources -avatar: +avatar: "/commons/avatar.jpg" # The URL of the site-wide social preview image used in SEO `og:image` meta tag. # It can be overridden by a customized `page.image` in front matter. @@ -110,11 +111,11 @@ comments: provider: # [disqus | utterances | giscus] # The provider options are as follows: disqus: - shortname: # fill with the Disqus shortname. › https://help.disqus.com/en/articles/1717111-what-s-a-shortname + shortname: xyz-ai # fill with the Disqus shortname. › https://help.disqus.com/en/articles/1717111-what-s-a-shortname # utterances settings › https://utteranc.es/ utterances: - repo: # / - issue_term: # < url | pathname | title | ...> + repo: zdongs/zdongs.github.io # / + issue_term: title # < url | pathname | title | ...> # Giscus options › https://giscus.app giscus: repo: # / @@ -168,6 +169,9 @@ collections: tabs: output: true sort_by: order + app: + output: true + permalink: /:name defaults: - scope: @@ -190,10 +194,6 @@ defaults: values: layout: page permalink: /:title/ - - scope: - path: assets/js/dist - values: - swcache: true sass: style: compressed diff --git a/_includes/analytics/cloudflare.html b/_includes/analytics/cloudflare.html new file mode 100644 index 0000000..1eeb1a9 --- /dev/null +++ b/_includes/analytics/cloudflare.html @@ -0,0 +1,7 @@ + + + diff --git a/_includes/analytics/goatcounter.html b/_includes/analytics/goatcounter.html new file mode 100644 index 0000000..3867fdb --- /dev/null +++ b/_includes/analytics/goatcounter.html @@ -0,0 +1,6 @@ + + diff --git a/_includes/analytics/google.html b/_includes/analytics/google.html new file mode 100644 index 0000000..d0aac65 --- /dev/null +++ b/_includes/analytics/google.html @@ -0,0 +1,13 @@ + + + diff --git a/_includes/analytics/matomo.html b/_includes/analytics/matomo.html new file mode 100644 index 0000000..72b2c46 --- /dev/null +++ b/_includes/analytics/matomo.html @@ -0,0 +1,14 @@ + + + diff --git a/_includes/analytics/umami.html b/_includes/analytics/umami.html new file mode 100644 index 0000000..bfcb1d0 --- /dev/null +++ b/_includes/analytics/umami.html @@ -0,0 +1,6 @@ + + diff --git a/_includes/comments.html b/_includes/comments.html new file mode 100644 index 0000000..fef135f --- /dev/null +++ b/_includes/comments.html @@ -0,0 +1,5 @@ + +{% if page.comments and site.comments.provider %} + {% capture path %}comments/{{ site.comments.provider }}.html{% endcapture %} + {% include {{ path }} %} +{% endif %} diff --git a/_includes/comments/disqus.html b/_includes/comments/disqus.html new file mode 100644 index 0000000..2b889a4 --- /dev/null +++ b/_includes/comments/disqus.html @@ -0,0 +1,50 @@ + + +
+

Comments powered by Disqus.

+
+ + diff --git a/_includes/comments/giscus.html b/_includes/comments/giscus.html new file mode 100644 index 0000000..f9becfe --- /dev/null +++ b/_includes/comments/giscus.html @@ -0,0 +1,71 @@ + + diff --git a/_includes/comments/utterances.html b/_includes/comments/utterances.html new file mode 100644 index 0000000..5dd78ed --- /dev/null +++ b/_includes/comments/utterances.html @@ -0,0 +1,49 @@ + + + + diff --git a/_includes/datetime.html b/_includes/datetime.html new file mode 100644 index 0000000..9f954b6 --- /dev/null +++ b/_includes/datetime.html @@ -0,0 +1,20 @@ + + +{% assign df_strftime = site.data.locales[include.lang].df.post.strftime | default: '%d/%m/%Y' %} +{% assign df_dayjs = site.data.locales[include.lang].df.post.dayjs | default: 'DD/MM/YYYY' %} + + diff --git a/_includes/embed/audio.html b/_includes/embed/audio.html new file mode 100644 index 0000000..cf928a7 --- /dev/null +++ b/_includes/embed/audio.html @@ -0,0 +1,35 @@ +{% assign src = include.src | strip %} +{% assign title = include.title | strip %} +{% assign types = include.types | default: '' | strip | split: '|' %} + +{% unless src contains '://' %} + {%- capture src -%} + {% include media-url.html src=src subpath=page.media_subpath %} + {%- endcapture -%} +{% endunless %} + +

+ + {% if title %} + {{ title }} + {% endif %} +

diff --git a/_includes/embed/bilibili.html b/_includes/embed/bilibili.html new file mode 100644 index 0000000..0aa5552 --- /dev/null +++ b/_includes/embed/bilibili.html @@ -0,0 +1,9 @@ + diff --git a/_includes/embed/twitch.html b/_includes/embed/twitch.html new file mode 100644 index 0000000..ed5ec83 --- /dev/null +++ b/_includes/embed/twitch.html @@ -0,0 +1,8 @@ + diff --git a/_includes/embed/video.html b/_includes/embed/video.html new file mode 100644 index 0000000..9b6918f --- /dev/null +++ b/_includes/embed/video.html @@ -0,0 +1,59 @@ +{% assign video_url = include.src %} +{% assign title = include.title %} +{% assign poster_url = include.poster %} +{% assign types = include.types | default: '' | strip | split: '|' %} + +{% unless video_url contains '://' %} + {%- capture video_url -%} + {% include media-url.html src=video_url subpath=page.media_subpath %} + {%- endcapture -%} +{% endunless %} + +{% if poster_url %} + {% unless poster_url contains '://' %} + {%- capture poster_url -%} + {% include media-url.html src=poster_url subpath=page.media_subpath %} + {%- endcapture -%} + {% endunless %} + {% assign poster = 'poster="' | append: poster_url | append: '"' %} +{% endif %} + +{% assign attributes = 'controls' %} + +{% if include.autoplay %} + {% assign attributes = attributes | append: ' ' | append: 'autoplay' %} +{% endif %} + +{% if include.loop %} + {% assign attributes = attributes | append: ' ' | append: 'loop' %} +{% endif %} + +{% if include.muted %} + {% assign attributes = attributes | append: ' ' | append: 'muted' %} +{% endif %} + +

+ + {% if title %} + {{ title }} + {% endif %} +

diff --git a/_includes/embed/youtube.html b/_includes/embed/youtube.html new file mode 100644 index 0000000..8f08002 --- /dev/null +++ b/_includes/embed/youtube.html @@ -0,0 +1,9 @@ + diff --git a/_includes/favicons.html b/_includes/favicons.html new file mode 100644 index 0000000..957c933 --- /dev/null +++ b/_includes/favicons.html @@ -0,0 +1,19 @@ + + +{% capture favicon_path %}{{ '/assets/img/favicons' | relative_url }}{% endcapture %} + + + + +{% if site.pwa.enabled %} + +{% endif %} + + + + + + diff --git a/_includes/footer.html b/_includes/footer.html new file mode 100644 index 0000000..1ba9b63 --- /dev/null +++ b/_includes/footer.html @@ -0,0 +1,49 @@ + + +
+

+ {{- '©' }} + + + {% if site.social.links %} + {{ site.social.name }}. + {% else %} + {{ site.social.name }}. + {% endif %} + + {% if site.data.locales[include.lang].copyright.brief %} + + {{- site.data.locales[include.lang].copyright.brief -}} + + {% endif %} +

+ +

+ {%- capture _platform -%} + Jekyll + {%- endcapture -%} + + {%- capture _theme -%} + Chirpy + {%- endcapture -%} + + {{ site.data.locales[include.lang].meta | replace: ':PLATFORM', _platform | replace: ':THEME', _theme }} +

+
diff --git a/_includes/head.html b/_includes/head.html new file mode 100644 index 0000000..fd260c0 --- /dev/null +++ b/_includes/head.html @@ -0,0 +1,107 @@ + + + + + + + + + {%- capture seo_tags -%} + {% seo title=false %} + {%- endcapture -%} + + + + {% if page.image %} + {% assign src = page.image.path | default: page.image %} + + {% unless src contains '://' %} + {%- capture img_url -%} + {% include media-url.html src=src subpath=page.media_subpath absolute=true %} + {%- endcapture -%} + + {%- capture old_url -%}{{ src | absolute_url }}{%- endcapture -%} + {%- capture new_url -%}{{ img_url }}{%- endcapture -%} + + {% assign seo_tags = seo_tags | replace: old_url, new_url %} + {% endunless %} + + {% elsif site.social_preview_image %} + {%- capture img_url -%} + {% include media-url.html src=site.social_preview_image absolute=true %} + {%- endcapture -%} + + {%- capture og_image -%} + + {%- endcapture -%} + + {%- capture twitter_image -%} + + + {%- endcapture -%} + + {% assign old_meta_clip = '' %} + {% assign new_meta_clip = og_image | append: twitter_image %} + {% assign seo_tags = seo_tags | replace: old_meta_clip, new_meta_clip %} + {% endif %} + + {{ seo_tags }} + + + {%- unless page.layout == 'home' -%} + {{ page.title | append: ' | ' }} + {%- endunless -%} + {{ site.title }} + + + {% include_cached favicons.html %} + + + {% unless site.assets.self_host.enabled %} + {% for hint in site.data.origin.cors.resource_hints %} + {% for link in hint.links %} + + {% endfor %} + {% endfor %} + {% endunless %} + + + {% unless jekyll.environment == 'production' %} + + {% endunless %} + + + + + + + + + + + + + {% if site.toc and page.toc %} + + {% endif %} + + {% if page.layout == 'post' or page.layout == 'page' or page.layout == 'home' %} + + {% endif %} + + {% if page.layout == 'page' or page.layout == 'post' %} + + + {% endif %} + + + + {% unless site.theme_mode %} + {% include mode-toggle.html %} + {% endunless %} + + {% include metadata-hook.html %} + diff --git a/_includes/js-selector.html b/_includes/js-selector.html new file mode 100644 index 0000000..9965107 --- /dev/null +++ b/_includes/js-selector.html @@ -0,0 +1,106 @@ + + + + +{% assign urls = site.data.origin[type].search.js %} + + + +{% if page.layout == 'post' or page.layout == 'page' or page.layout == 'home' %} + {% assign urls = urls | append: ',' | append: site.data.origin[type]['lazy-polyfill'].js %} + + {% unless page.layout == 'home' %} + + {% assign urls = urls + | append: ',' + | append: site.data.origin[type].glightbox.js + | append: ',' + | append: site.data.origin[type].clipboard.js + %} + {% endunless %} +{% endif %} + +{% if page.layout == 'home' + or page.layout == 'post' + or page.layout == 'archives' + or page.layout == 'category' + or page.layout == 'tag' +%} + {% assign locale = include.lang | split: '-' | first %} + + {% assign urls = urls + | append: ',' + | append: site.data.origin[type].dayjs.js.common + | append: ',' + | append: site.data.origin[type].dayjs.js.locale + | replace: ':LOCALE', locale + | append: ',' + | append: site.data.origin[type].dayjs.js.relativeTime + | append: ',' + | append: site.data.origin[type].dayjs.js.localizedFormat + %} +{% endif %} + +{% if page.content contains ' + +{% if page.math %} + + + + +{% endif %} + + +{% if page.layout == 'post' %} + {% assign provider = site.pageviews.provider %} + + {% if provider and provider != empty %} + {% case provider %} + {% when 'goatcounter' %} + {% if site.analytics[provider].id != empty and site.analytics[provider].id %} + {% include pageviews/{{ provider }}.html %} + {% endif %} + {% endcase %} + {% endif %} +{% endif %} + +{% if page.mermaid %} + {% include mermaid.html %} +{% endif %} + +{% if jekyll.environment == 'production' %} + + {% if site.pwa.enabled %} + + {% endif %} + + + {% for analytics in site.analytics %} + {% capture str %}{{ analytics }}{% endcapture %} + {% assign type = str | split: '{' | first %} + {% if site.analytics[type].id and site.analytics[type].id != empty %} + {% include analytics/{{ type }}.html %} + {% endif %} + {% endfor %} +{% endif %} diff --git a/_includes/jsdelivr-combine.html b/_includes/jsdelivr-combine.html new file mode 100644 index 0000000..cffa699 --- /dev/null +++ b/_includes/jsdelivr-combine.html @@ -0,0 +1,26 @@ +{% assign urls = include.urls | split: ',' %} + +{% assign combined_urls = nil %} + +{% assign domain = 'https://cdn.jsdelivr.net/' %} + +{% for url in urls %} + {% if url contains domain %} + {% assign url_snippet = url | slice: domain.size, url.size %} + + {% if combined_urls %} + {% assign combined_urls = combined_urls | append: ',' | append: url_snippet %} + {% else %} + {% assign combined_urls = domain | append: 'combine/' | append: url_snippet %} + {% endif %} + + {% elsif url contains '//' %} + + {% else %} + + {% endif %} +{% endfor %} + +{% if combined_urls %} + +{% endif %} diff --git a/_includes/lang.html b/_includes/lang.html new file mode 100644 index 0000000..34b50df --- /dev/null +++ b/_includes/lang.html @@ -0,0 +1,10 @@ +{% comment %} + Detect appearance language and return it through variable "lang" +{% endcomment %} +{% if site.data.locales[page.lang] %} + {% assign lang = page.lang %} +{% elsif site.data.locales[site.lang] %} + {% assign lang = site.lang %} +{% else %} + {% assign lang = 'en' %} +{% endif %} diff --git a/_includes/language-alias.html b/_includes/language-alias.html new file mode 100644 index 0000000..abfa7ba --- /dev/null +++ b/_includes/language-alias.html @@ -0,0 +1,70 @@ +{% comment %} + + Convert the alias of the syntax language to the official name + + See: + +{% endcomment %} + +{% assign _lang = include.language | default: '' %} + +{% case _lang %} + {% when 'actionscript', 'as', 'as3' %} + {{ 'ActionScript' }} + {% when 'applescript' %} + {{ 'AppleScript' }} + {% when 'brightscript', 'bs', 'brs' %} + {{ 'BrightScript' }} + {% when 'cfscript', 'cfc' %} + {{ 'CFScript' }} + {% when 'coffeescript', 'coffee', 'coffee-script' %} + {{ 'CoffeeScript' }} + {% when 'cs', 'csharp' %} + {{ 'C#' }} + {% when 'erl' %} + {{ 'Erlang' }} + {% when 'graphql' %} + {{ 'GraphQL' }} + {% when 'haskell', 'hs' %} + {{ 'Haskell' }} + {% when 'javascript', 'js' %} + {{ 'JavaScript' }} + {% when 'make', 'mf', 'gnumake', 'bsdmake' %} + {{ 'Makefile' }} + {% when 'md', 'mkd' %} + {{ 'Markdown' }} + {% when 'm' %} + {{ 'Matlab' }} + {% when 'objective_c', 'objc', 'obj-c', 'obj_c', 'objectivec' %} + {{ 'Objective-C' }} + {% when 'perl', 'pl' %} + {{ 'Perl' }} + {% when 'php','php3','php4','php5' %} + {{ 'PHP' }} + {% when 'py' %} + {{ 'Python' }} + {% when 'rb' %} + {{ 'Ruby' }} + {% when 'rs','no_run','ignore','should_panic' %} + {{ 'Rust' }} + {% when 'bash', 'zsh', 'ksh', 'sh' %} + {{ 'Shell' }} + {% when 'st', 'squeak' %} + {{ 'Smalltalk' }} + {% when 'tex'%} + {{ 'TeX' }} + {% when 'latex' %} + {{ 'LaTex' }} + {% when 'ts', 'typescript' %} + {{ 'TypeScript' }} + {% when 'vb', 'visualbasic' %} + {{ 'Visual Basic' }} + {% when 'vue', 'vuejs' %} + {{ 'Vue.js' }} + {% when 'yml' %} + {{ 'YAML' }} + {% when 'css', 'html', 'scss', 'ssh', 'toml', 'xml', 'yaml', 'json' %} + {{ _lang | upcase }} + {% else %} + {{ _lang | capitalize }} +{% endcase %} diff --git a/_includes/mathjax_support.html b/_includes/mathjax_support.html new file mode 100644 index 0000000..cdbdec4 --- /dev/null +++ b/_includes/mathjax_support.html @@ -0,0 +1,30 @@ + + + \ No newline at end of file diff --git a/_includes/media-url.html b/_includes/media-url.html new file mode 100644 index 0000000..ea41075 --- /dev/null +++ b/_includes/media-url.html @@ -0,0 +1,37 @@ +{%- comment -%} + Generate media resource final URL based on `site.cdn`, `page.media_subpath` + + Arguments: + src - required, basic media resources path + subpath - optional, relative path of media resources + absolute - optional, boolean, if true, generate absolute URL + + Return: + media resources URL +{%- endcomment -%} + +{% assign url = include.src %} + +{%- if url -%} + {% unless url contains ':' %} + {%- comment -%} Add media resources subpath prefix {%- endcomment -%} + {% assign url = include.subpath | default: '' | append: '/' | append: url %} + + {%- comment -%} Prepend CND URL {%- endcomment -%} + {% if site.cdn %} + {% assign url = site.cdn | append: '/' | append: url %} + {% endif %} + + {% assign url = url | replace: '///', '/' | replace: '//', '/' | replace: ':/', '://' %} + + {% unless url contains '://' %} + {% if include.absolute %} + {% assign url = site.url | append: site.baseurl | append: url %} + {% else %} + {% assign url = site.baseurl | append: url %} + {% endif %} + {% endunless %} + {% endunless %} +{%- endif -%} + +{{- url -}} diff --git a/_includes/mermaid.html b/_includes/mermaid.html new file mode 100644 index 0000000..a3a83ed --- /dev/null +++ b/_includes/mermaid.html @@ -0,0 +1,62 @@ + + diff --git a/_includes/metadata-hook.html b/_includes/metadata-hook.html new file mode 100644 index 0000000..fd7e9bd --- /dev/null +++ b/_includes/metadata-hook.html @@ -0,0 +1 @@ + diff --git a/_includes/mode-toggle.html b/_includes/mode-toggle.html new file mode 100644 index 0000000..113ec37 --- /dev/null +++ b/_includes/mode-toggle.html @@ -0,0 +1,116 @@ + + + diff --git a/_includes/no-linenos.html b/_includes/no-linenos.html new file mode 100644 index 0000000..8500693 --- /dev/null +++ b/_includes/no-linenos.html @@ -0,0 +1,10 @@ +{% comment %} + Remove the line number of the code snippet. +{% endcomment %} + +{% assign content = include.content %} + +{% if content contains '
' %}
+  {% assign content = content | replace: '
', '' %}
+{% endif %}
diff --git a/_includes/notification.html b/_includes/notification.html
new file mode 100644
index 0000000..80049b0
--- /dev/null
+++ b/_includes/notification.html
@@ -0,0 +1,24 @@
+
diff --git a/_includes/origin-type.html b/_includes/origin-type.html
new file mode 100644
index 0000000..7f72012
--- /dev/null
+++ b/_includes/origin-type.html
@@ -0,0 +1,13 @@
+{% comment %} Site static assets origin type {% endcomment %}
+
+{% assign type = 'cors' %}
+
+{% if site.assets.self_host.enabled %}
+  {% if site.assets.self_host.env %}
+    {% if site.assets.self_host.env == jekyll.environment %}
+      {% assign type = 'basic' %}
+    {% endif %}
+  {% else %}
+    {% assign type = 'basic' %}
+  {% endif %}
+{% endif %}
diff --git a/_includes/pageviews/goatcounter.html b/_includes/pageviews/goatcounter.html
new file mode 100644
index 0000000..e62fd69
--- /dev/null
+++ b/_includes/pageviews/goatcounter.html
@@ -0,0 +1,19 @@
+
+
diff --git a/_includes/post-description.html b/_includes/post-description.html
new file mode 100644
index 0000000..6c40036
--- /dev/null
+++ b/_includes/post-description.html
@@ -0,0 +1,16 @@
+{%- comment -%}
+  Get post description or generate it from the post content.
+{%- endcomment -%}
+
+{%- assign max_length = include.max_length | default: 200 -%}
+
+{%- capture description -%}
+{%- if post.description -%}
+  {{- post.description -}}
+{%- else -%}
+  {%- include no-linenos.html content=post.content -%}
+  {{- content | markdownify | strip_html -}}
+{%- endif -%}
+{%- endcapture -%}
+
+{{- description | strip | truncate: max_length | escape -}}
diff --git a/_includes/post-nav.html b/_includes/post-nav.html
new file mode 100644
index 0000000..736bec3
--- /dev/null
+++ b/_includes/post-nav.html
@@ -0,0 +1,34 @@
+
+
+
diff --git a/_includes/post-paginator.html b/_includes/post-paginator.html
new file mode 100644
index 0000000..c74e978
--- /dev/null
+++ b/_includes/post-paginator.html
@@ -0,0 +1,91 @@
+
+
+
+
diff --git a/_includes/post-sharing.html b/_includes/post-sharing.html
new file mode 100644
index 0000000..d894199
--- /dev/null
+++ b/_includes/post-sharing.html
@@ -0,0 +1,52 @@
+
+
+
diff --git a/_includes/read-time.html b/_includes/read-time.html
new file mode 100644
index 0000000..9952410
--- /dev/null
+++ b/_includes/read-time.html
@@ -0,0 +1,37 @@
+
+
+{% assign words = include.content | strip_html | number_of_words: 'auto' %}
+
+
+
+{% assign wpm = 180 %}
+{% assign min_time = 1 %}
+
+{% assign read_time = words | divided_by: wpm %}
+
+{% unless read_time > 0 %}
+  {% assign read_time = min_time %}
+{% endunless %}
+
+{% capture read_prompt %}
+  {{- site.data.locales[include.lang].post.read_time.prompt -}}
+{% endcapture %}
+
+
+
+  
+    {{- read_time -}}
+    {{ ' ' }}
+    {{- site.data.locales[include.lang].post.read_time.unit -}}
+  
+  {%- if include.prompt -%}
+    {%- assign _prompt_words = read_prompt | number_of_words: 'auto' -%}
+    {%- unless _prompt_words > 1 -%}{{ ' ' }}{%- endunless -%}
+    {{ read_prompt }}
+  {%- endif -%}
+
diff --git a/_includes/refactor-content.html b/_includes/refactor-content.html
new file mode 100644
index 0000000..8d298cd
--- /dev/null
+++ b/_includes/refactor-content.html
@@ -0,0 +1,255 @@
+
+
+{% assign _content = include.content %}
+
+
+
+{% if _content contains '', ''
+    | replace: '
', '' + | replace: '
', '
' + %} +{% endif %} + + + +{% if _content contains '
' %}
+  {% assign _content = _content
+    | replace: '
', '' + %} +{% endif %} + + + +{% if _content contains '', + '' + | replace: '', + '' + %} +{% endif %} + + + +{% assign IMG_TAG = '' | first %} + {% assign _right = _img_snippet | remove: _left %} + + {% unless _left contains 'src=' %} + {% continue %} + {% endunless %} + + {% assign _left = _left | remove: ' /' | replace: ' w=', ' width=' | replace: ' h=', ' height=' %} + {% assign _attrs = _left | split: '" ' %} + + {% assign _src = null %} + {% assign _lqip = null %} + {% assign _class = null %} + + {% for _attr in _attrs %} + {% unless _attr contains '=' %} + {% continue %} + {% endunless %} + + {% assign _pair = _attr | split: '="' %} + {% capture _key %}{{ _pair | first }}{% endcapture %} + {% capture _value %}{{ _pair | last | remove: '"' }}{% endcapture %} + + {% case _key %} + {% when 'src' %} + {% assign _src = _value %} + {% when 'lqip' %} + {% assign _lqip = _value %} + {% when 'class' %} + {% assign _class = _value %} + {% endcase %} + {% endfor %} + + + {% if _class %} + {% capture _old_class %}class="{{ _class }}"{% endcapture %} + {% assign _left = _left | remove: _old_class %} + {% endif %} + + {% assign _final_src = null %} + {% assign _lazyload = true %} + + {%- capture _img_url -%} + {% include media-url.html src=_src subpath=page.media_subpath %} + {%- endcapture -%} + + {% assign _path_prefix = _img_url | remove: _src %} + + {% unless _src contains '//' %} + {% assign _final_src = _path_prefix | append: _src %} + {% assign _src_alt = 'src="' | append: _path_prefix %} + {% assign _left = _left | replace: 'src="', _src_alt %} + {% endunless %} + + {% if _lqip %} + {% assign _lazyload = false %} + {% assign _class = _class | append: ' blur' %} + + {% unless _lqip contains 'data:' %} + {% assign _lqip_alt = 'lqip="' | append: _path_prefix %} + {% assign _left = _left | replace: 'lqip="', _lqip_alt %} + {% endunless %} + + + {% assign _left = _left | replace: 'src=', 'data-src=' | replace: ' lqip=', ' data-lqip="true" src=' %} + + {% else %} + {% assign _class = _class | append: ' shimmer' %} + {% endif %} + + + {% if _lazyload %} + {% assign _left = _left | append: ' loading="lazy"' %} + {% endif %} + + {% if page.layout == 'home' %} + + {% assign _wrapper_start = '
' %} + + {% assign _img_content = _img_content | append: _wrapper_start %} + {% assign _right = _right | prepend: '>` is wrapped by `` --> + {% assign _parent = _right | slice: 1, 4 %} + + {% if _parent == '' %} + + {% assign _size = _img_content | size | minus: 1 %} + {% capture _class %} + class="img-link{% unless _lqip %} shimmer{% endunless %}" + {% endcapture %} + {% assign _img_content = _img_content | slice: 0, _size | append: _class | append: '>' %} + + {% else %} + + {% assign _wrapper_start = _final_src + | default: _src + | prepend: '' + %} + + {% assign _img_content = _img_content | append: _wrapper_start %} + {% assign _right = '> + {% assign _img_content = _img_content | append: IMG_TAG | append: _left | append: _right %} + {% endfor %} + + {% if _img_content %} + {% assign _content = _img_content %} + {% endif %} +{% endif %} + + + +{% if _content contains '
' %} + {% assign _code_spippets = _content | split: '
' %} + {% assign _new_content = '' %} + + {% for _snippet in _code_spippets %} + {% if forloop.last %} + {% assign _new_content = _new_content | append: _snippet %} + + {% else %} + {% assign _left = _snippet | split: '><' | last %} + + {% if _left contains 'file="' %} + {% assign _label_text = _left | split: 'file="' | last | split: '"' | first %} + {% assign _label_icon = 'far fa-file-code fa-fw' %} + {% else %} + {% assign _lang = _left | split: 'language-' | last | split: ' ' | first %} + {% capture _label_text %}{% include language-alias.html language=_lang %}{% endcapture %} + {% assign _label_icon = 'fas fa-code fa-fw small' %} + {% endif %} + + {% capture _label %} + + {% endcapture %} + + {% assign _new_content = _new_content + | append: _snippet + | append: '
' + | append: _label + | append: '
' + | append: '
' + %} + {% endif %} + {% endfor %} + + {% assign _content = _new_content %} +{% endif %} + + + +{% assign heading_levels = '2,3,4,5' | split: ',' %} +{% assign _heading_content = _content %} + +{% for level in heading_levels %} + {% assign mark_start = '' + %} + + {% assign left = snippet | split: mark_end | first %} + {% assign right = snippet | slice: left.size, snippet.size %} + {% assign left = left | replace_first: '">', '">' | append: '' %} + + {% assign _new_content = _new_content | append: mark_start | append: left | append: anchor | append: right %} + {% endfor %} + + {% assign _heading_content = _new_content %} + {% endif %} +{% endfor %} + +{% assign _content = _heading_content %} + + +{{ _content }} diff --git a/_includes/related-posts.html b/_includes/related-posts.html new file mode 100644 index 0000000..37a295b --- /dev/null +++ b/_includes/related-posts.html @@ -0,0 +1,94 @@ + + + +{% assign TOTAL_SIZE = 3 %} + + +{% assign TAG_SCORE = 1 %} + + +{% assign CATEGORY_SCORE = 0.5 %} + +{% assign SEPARATOR = ':' %} + +{% assign match_posts = '' | split: '' %} + +{% for category in page.categories %} + {% assign match_posts = match_posts | push: site.categories[category] | uniq %} +{% endfor %} + +{% for tag in page.tags %} + {% assign match_posts = match_posts | push: site.tags[tag] | uniq %} +{% endfor %} + +{% assign match_posts = match_posts | reverse %} +{% assign last_index = match_posts.size | minus: 1 %} +{% assign score_list = '' | split: '' %} + +{% for i in (0..last_index) %} + {% assign post = match_posts[i] %} + + {% if post.url == page.url %} + {% continue %} + {% endif %} + + {% assign score = 0 %} + + {% for tag in post.tags %} + {% if page.tags contains tag %} + {% assign score = score | plus: TAG_SCORE %} + {% endif %} + {% endfor %} + + {% for category in post.categories %} + {% if page.categories contains category %} + {% assign score = score | plus: CATEGORY_SCORE %} + {% endif %} + {% endfor %} + + {% if score > 0 %} + {% capture score_item %}{{ score }}{{ SEPARATOR }}{{ i }}{% endcapture %} + {% assign score_list = score_list | push: score_item %} + {% endif %} +{% endfor %} + +{% assign index_list = '' | split: '' %} + +{% if score_list.size > 0 %} + {% assign score_list = score_list | sort | reverse %} + {% for entry in score_list limit: TOTAL_SIZE %} + {% assign index = entry | split: SEPARATOR | last %} + {% assign index_list = index_list | push: index %} + {% endfor %} +{% endif %} + +{% assign relate_posts = '' | split: '' %} + +{% for index in index_list %} + {% assign i = index | to_integer %} + {% assign relate_posts = relate_posts | push: match_posts[i] %} +{% endfor %} + +{% if relate_posts.size > 0 %} + + +{% endif %} diff --git a/_includes/search-loader.html b/_includes/search-loader.html new file mode 100644 index 0000000..2582580 --- /dev/null +++ b/_includes/search-loader.html @@ -0,0 +1,47 @@ + + +{% capture result_elem %} +
+
+

{title}

+ +
+

{snippet}

+
+{% endcapture %} + +{% capture not_found %}

{{ site.data.locales[include.lang].search.no_results }}

{% endcapture %} + + diff --git a/_includes/search-results.html b/_includes/search-results.html new file mode 100644 index 0000000..00a3182 --- /dev/null +++ b/_includes/search-results.html @@ -0,0 +1,10 @@ + + +
+
+
+ {% include_cached trending-tags.html %} +
+
+
+
diff --git a/_includes/sidebar.html b/_includes/sidebar.html new file mode 100644 index 0000000..4f0bb8c --- /dev/null +++ b/_includes/sidebar.html @@ -0,0 +1,99 @@ + + + + diff --git a/_includes/toc.html b/_includes/toc.html new file mode 100644 index 0000000..15eb934 --- /dev/null +++ b/_includes/toc.html @@ -0,0 +1,13 @@ +{% assign enable_toc = false %} +{% if site.toc and page.toc %} + {% if page.content contains ' +

{{- site.data.locales[include.lang].panel.toc -}}

+ + +{% endif %} diff --git a/_includes/topbar.html b/_includes/topbar.html new file mode 100644 index 0000000..fd68d1f --- /dev/null +++ b/_includes/topbar.html @@ -0,0 +1,77 @@ + + +
+
+ + + + + +
+ {% if page.layout == 'home' %} + {{- site.data.locales[include.lang].title | default: site.title -}} + {% elsif page.collection == 'tabs' or page.layout == 'page' %} + {%- capture tab_key -%}{{ page.url | split: '/' }}{%- endcapture -%} + {{- site.data.locales[include.lang].tabs[tab_key] | default: page.title -}} + {% else %} + {{- site.data.locales[include.lang].layout[page.layout] | default: page.layout | capitalize -}} + {% endif %} +
+ + + + + + + + +
+
diff --git a/_includes/trending-tags.html b/_includes/trending-tags.html new file mode 100644 index 0000000..57369f0 --- /dev/null +++ b/_includes/trending-tags.html @@ -0,0 +1,46 @@ + + +{% assign MAX = 10 %} + +{% assign size_list = '' | split: '' %} +{% assign tag_list = '' | split: '' %} + +{% for tag in site.tags %} + {% assign size = tag | last | size %} + {% assign size_list = size_list | push: size %} + + {% assign tag_str = tag | first | append: '::' | append: size %} + {% assign tag_list = tag_list | push: tag_str %} +{% endfor %} + +{% assign size_list = size_list | sort | reverse %} + +{% assign tag_list = tag_list | sort_natural %} + +{% assign trending_tags = '' | split: '' %} + +{% for size in size_list limit: MAX %} + {% for tag_str in tag_list %} + {% assign tag = tag_str | split: '::' %} + {% assign tag_name = tag | first %} + {% assign tag_size = tag | last | plus: 0 %} + {% if tag_size == size %} + {% unless trending_tags contains tag_name %} + {% assign trending_tags = trending_tags | push: tag_name %} + {% break %} + {% endunless %} + {% endif %} + {% endfor %} +{% endfor %} + +{% if trending_tags.size > 0 %} +
+

{{- site.data.locales[include.lang].panel.trending_tags -}}

+
+ {% for tag_name in trending_tags %} + {% assign url = tag_name | slugify | url_encode | prepend: '/tags/' | append: '/' %} + + {% endfor %} +
+
+{% endif %} diff --git a/_includes/update-list.html b/_includes/update-list.html new file mode 100644 index 0000000..93684c3 --- /dev/null +++ b/_includes/update-list.html @@ -0,0 +1,40 @@ + + +{% assign MAX_SIZE = 5 %} + +{% assign all_list = '' | split: '' %} + +{% for post in site.posts %} + {% assign datetime = post.last_modified_at | default: post.date %} + + {% capture elem %} + {{- datetime | date: "%Y%m%d%H%M%S" -}}::{{- forloop.index0 -}} + {% endcapture %} + + {% assign all_list = all_list | push: elem %} +{% endfor %} + +{% assign all_list = all_list | sort | reverse %} + +{% assign update_list = '' | split: '' %} + +{% for entry in all_list limit: MAX_SIZE %} + {% assign update_list = update_list | push: entry %} +{% endfor %} + +{% if update_list.size > 0 %} +
+

{{- site.data.locales[include.lang].panel.lastmod -}}

+
    + {% for item in update_list %} + {% assign index = item | split: '::' | last | plus: 0 %} + {% assign post = site.posts[index] %} + {% assign url = post.url | relative_url %} +
  • + {{ post.title }} +
  • + {% endfor %} +
+
+ +{% endif %} diff --git a/_layouts/archives.html b/_layouts/archives.html new file mode 100644 index 0000000..4f7ad7d --- /dev/null +++ b/_layouts/archives.html @@ -0,0 +1,35 @@ +--- +layout: page +# The Archives of posts. +--- + +{% include lang.html %} + +{% assign df_strftime_m = site.data.locales[lang].df.archives.strftime | default: '/ %m' %} +{% assign df_dayjs_m = site.data.locales[lang].df.archives.dayjs | default: '/ MM' %} + +
+ {% for post in site.posts %} + {% assign cur_year = post.date | date: '%Y' %} + + {% if cur_year != last_year %} + {% unless forloop.first %}{% endunless %} + + + {{ '
    ' }} + + {% assign last_year = cur_year %} + {% endif %} + +
  • + {% assign ts = post.date | date: '%s' %} + {{ post.date | date: '%d' }} + + {{ post.date | date: df_strftime_m }} + + {{ post.title }} +
  • + + {% if forloop.last %}
{% endif %} + {% endfor %} +
diff --git a/_layouts/categories.html b/_layouts/categories.html new file mode 100644 index 0000000..0515097 --- /dev/null +++ b/_layouts/categories.html @@ -0,0 +1,138 @@ +--- +layout: page +# All the Categories of posts +--- + +{% include lang.html %} + +{% assign HEAD_PREFIX = 'h_' %} +{% assign LIST_PREFIX = 'l_' %} + +{% assign group_index = 0 %} + +{% assign sort_categories = site.categories | sort %} + +{% for category in sort_categories %} + {% assign category_name = category | first %} + {% assign posts_of_category = category | last %} + {% assign first_post = posts_of_category | first %} + + {% if category_name == first_post.categories[0] %} + {% assign sub_categories = '' | split: '' %} + + {% for post in posts_of_category %} + {% assign second_category = post.categories[1] %} + {% if second_category %} + {% unless sub_categories contains second_category %} + {% assign sub_categories = sub_categories | push: second_category %} + {% endunless %} + {% endif %} + {% endfor %} + + {% assign sub_categories = sub_categories | sort %} + {% assign sub_categories_size = sub_categories | size %} + +
+ +
+ + + + {% capture _category_url %}/categories/{{ category_name | slugify | url_encode }}/{% endcapture %} + {{ category_name }} + + + {% assign top_posts_size = site.categories[category_name] | size %} + + {% if sub_categories_size > 0 %} + {{ sub_categories_size }} + {% if sub_categories_size > 1 %} + {{ + site.data.locales[lang].categories.category_measure.plural + | default: site.data.locales[lang].categories.category_measure + }} + {% else %} + {{ + site.data.locales[lang].categories.category_measure.singular + | default: site.data.locales[lang].categories.category_measure + }} + {% endif -%} + , + {% endif %} + + {{ top_posts_size }} + + {% if top_posts_size > 1 %} + {{ + site.data.locales[lang].categories.post_measure.plural + | default: site.data.locales[lang].categories.post_measure + }} + {% else %} + {{ + site.data.locales[lang].categories.post_measure.singular + | default: site.data.locales[lang].categories.post_measure + }} + {% endif %} + + + + + {% if sub_categories_size > 0 %} + + + + {% else %} + + + + {% endif %} +
+ + + + {% if sub_categories_size > 0 %} +
+
    + {% for sub_category in sub_categories %} +
  • + + + {% capture _sub_ctg_url %}/categories/{{ sub_category | slugify | url_encode }}/{% endcapture %} + {{ sub_category }} + + {% assign posts_size = site.categories[sub_category] | size %} + + {{ posts_size }} + + {% if posts_size > 1 %} + {{ + site.data.locales[lang].categories.post_measure.plural + | default: site.data.locales[lang].categories.post_measure + }} + {% else %} + {{ + site.data.locales[lang].categories.post_measure.singular + | default: site.data.locales[lang].categories.post_measure + }} + {% endif %} + +
  • + {% endfor %} +
+
+ {% endif %} +
+ + + {% assign group_index = group_index | plus: 1 %} + {% endif %} +{% endfor %} diff --git a/_layouts/category.html b/_layouts/category.html new file mode 100644 index 0000000..b064f27 --- /dev/null +++ b/_layouts/category.html @@ -0,0 +1,24 @@ +--- +layout: page +# The Category layout +--- + +{% include lang.html %} + +
+

+ + {{ page.title }} + {{ page.posts | size }} +

+ +
    + {% for post in page.posts %} +
  • + {{ post.title }} + + {% include datetime.html date=post.date class='text-muted small text-nowrap' lang=lang %} +
  • + {% endfor %} +
+
diff --git a/_layouts/compress.html b/_layouts/compress.html new file mode 100644 index 0000000..2779e92 --- /dev/null +++ b/_layouts/compress.html @@ -0,0 +1,10 @@ +--- +# Jekyll layout that compresses HTML +# v3.2.0 +# http://jch.penibelst.de/ +# © 2014–2015 Anatol Broder +# MIT License +--- + +{% capture _LINE_FEED %} +{% endcapture %}{% if site.compress_html.ignore.envs contains jekyll.environment or site.compress_html.ignore.envs == "all" or page.compress_html == false %}{{ content }}{% else %}{% capture _content %}{{ content }}{% endcapture %}{% assign _profile = site.compress_html.profile %}{% if site.compress_html.endings == "all" %}{% assign _endings = "html head body li dt dd optgroup option colgroup caption thead tbody tfoot tr td th" | split: " " %}{% else %}{% assign _endings = site.compress_html.endings %}{% endif %}{% for _element in _endings %}{% capture _end %}{% endcapture %}{% assign _content = _content | remove: _end %}{% endfor %}{% if _profile and _endings %}{% assign _profile_endings = _content | size | plus: 1 %}{% endif %}{% for _element in site.compress_html.startings %}{% capture _start %}<{{ _element }}>{% endcapture %}{% assign _content = _content | remove: _start %}{% endfor %}{% if _profile and site.compress_html.startings %}{% assign _profile_startings = _content | size | plus: 1 %}{% endif %}{% if site.compress_html.comments == "all" %}{% assign _comments = "" | split: " " %}{% else %}{% assign _comments = site.compress_html.comments %}{% endif %}{% if _comments.size == 2 %}{% capture _comment_befores %}.{{ _content }}{% endcapture %}{% assign _comment_befores = _comment_befores | split: _comments.first %}{% for _comment_before in _comment_befores %}{% if forloop.first %}{% continue %}{% endif %}{% capture _comment_outside %}{% if _carry %}{{ _comments.first }}{% endif %}{{ _comment_before }}{% endcapture %}{% capture _comment %}{% unless _carry %}{{ _comments.first }}{% endunless %}{{ _comment_outside | split: _comments.last | first }}{% if _comment_outside contains _comments.last %}{{ _comments.last }}{% assign _carry = false %}{% else %}{% assign _carry = true %}{% endif %}{% endcapture %}{% assign _content = _content | remove_first: _comment %}{% endfor %}{% if _profile %}{% assign _profile_comments = _content | size | plus: 1 %}{% endif %}{% endif %}{% assign _pre_befores = _content | split: "" %}{% assign _pres_after = "" %}{% if _pres.size != 0 %}{% if site.compress_html.blanklines %}{% assign _lines = _pres.last | split: _LINE_FEED %}{% capture _pres_after %}{% for _line in _lines %}{% assign _trimmed = _line | split: " " | join: " " %}{% if _trimmed != empty or forloop.last %}{% unless forloop.first %}{{ _LINE_FEED }}{% endunless %}{{ _line }}{% endif %}{% endfor %}{% endcapture %}{% else %}{% assign _pres_after = _pres.last | split: " " | join: " " %}{% endif %}{% endif %}{% capture _content %}{{ _content }}{% if _pre_before contains "
" %}{% endif %}{% unless _pre_before contains "
" and _pres.size == 1 %}{{ _pres_after }}{% endunless %}{% endcapture %}{% endfor %}{% if _profile %}{% assign _profile_collapse = _content | size | plus: 1 %}{% endif %}{% if site.compress_html.clippings == "all" %}{% assign _clippings = "html head title base link meta style body article section nav aside h1 h2 h3 h4 h5 h6 hgroup header footer address p hr blockquote ol ul li dl dt dd figure figcaption main div table caption colgroup col tbody thead tfoot tr td th" | split: " " %}{% else %}{% assign _clippings = site.compress_html.clippings %}{% endif %}{% for _element in _clippings %}{% assign _edges = " ;; ;" | replace: "e", _element | split: ";" %}{% assign _content = _content | replace: _edges[0], _edges[1] | replace: _edges[2], _edges[3] | replace: _edges[4], _edges[5] %}{% endfor %}{% if _profile and _clippings %}{% assign _profile_clippings = _content | size | plus: 1 %}{% endif %}{{ _content }}{% if _profile %}
Step Bytes
raw {{ content | size }}{% if _profile_endings %}
endings {{ _profile_endings }}{% endif %}{% if _profile_startings %}
startings {{ _profile_startings }}{% endif %}{% if _profile_comments %}
comments {{ _profile_comments }}{% endif %}{% if _profile_collapse %}
collapse {{ _profile_collapse }}{% endif %}{% if _profile_clippings %}
clippings {{ _profile_clippings }}{% endif %}
{% endif %}{% endif %} diff --git a/_layouts/default.html b/_layouts/default.html new file mode 100644 index 0000000..9219db7 --- /dev/null +++ b/_layouts/default.html @@ -0,0 +1,82 @@ +--- +layout: compress +--- + + + +{% include origin-type.html %} + +{% include lang.html %} + +{% if site.theme_mode %} + {% capture prefer_mode %}data-mode="{{ site.theme_mode }}"{% endcapture %} +{% endif %} + + + + {% include head.html %} + {% include mathjax_support.html %} + + {% include sidebar.html lang=lang %} + +
+
+ {% include topbar.html lang=lang %} + +
+
+ {% if layout.refactor or layout.layout == 'default' %} + {% include refactor-content.html content=content lang=lang %} + {% else %} + {{ content }} + {% endif %} +
+ + + +
+ +
+ +
+ {% for _include in layout.tail_includes %} + {% assign _include_path = _include | append: '.html' %} + {% include {{ _include_path }} lang=lang %} + {% endfor %} + + {% include_cached footer.html lang=lang %} +
+
+ + {% include_cached search-results.html lang=lang %} +
+ + +
+ +
+ + {% if site.pwa.enabled %} + {% include_cached notification.html lang=lang %} + {% endif %} + + + {% include js-selector.html lang=lang %} + + {% include_cached search-loader.html lang=lang %} + + diff --git a/_layouts/home.html b/_layouts/home.html new file mode 100644 index 0000000..e44efe8 --- /dev/null +++ b/_layouts/home.html @@ -0,0 +1,115 @@ +--- +layout: default +refactor: true +--- + +{% include lang.html %} + +{% assign pinned = site.posts | where: 'pin', 'true' %} +{% assign default = site.posts | where_exp: 'item', 'item.pin != true and item.hidden != true' %} + +{% assign posts = '' | split: '' %} + + + +{% assign offset = paginator.page | minus: 1 | times: paginator.per_page %} +{% assign pinned_num = pinned.size | minus: offset %} + +{% if pinned_num > 0 %} + {% for i in (offset..pinned.size) limit: pinned_num %} + {% assign posts = posts | push: pinned[i] %} + {% endfor %} +{% else %} + {% assign pinned_num = 0 %} +{% endif %} + + + +{% assign default_beg = offset | minus: pinned.size %} + +{% if default_beg < 0 %} + {% assign default_beg = 0 %} +{% endif %} + +{% assign default_num = paginator.posts | size | minus: pinned_num %} +{% assign default_end = default_beg | plus: default_num | minus: 1 %} + +{% if default_num > 0 %} + {% for i in (default_beg..default_end) %} + {% assign posts = posts | push: default[i] %} + {% endfor %} +{% endif %} + +
+ {% for post in posts %} +
+ + {% assign card_body_col = '12' %} + + {% if post.image %} + {% assign src = post.image.path | default: post.image %} + {% unless src contains '//' %} + {% assign src = post.media_subpath | append: '/' | append: src | replace: '//', '/' %} + {% endunless %} + + {% assign alt = post.image.alt | xml_escape | default: 'Preview Image' %} + + {% assign lqip = null %} + + {% if post.image.lqip %} + {% capture lqip %}lqip="{{ post.image.lqip }}"{% endcapture %} + {% endif %} + +
+ {{ alt }} +
+ + {% assign card_body_col = '7' %} + {% endif %} + +
+
+

{{ post.title }}

+ +
+

{% include post-description.html %}

+
+ + + +
+ +
+
+
+ {% endfor %} +
+ + +{% if paginator.total_pages > 1 %} + {% include post-paginator.html %} +{% endif %} diff --git a/_layouts/page.html b/_layouts/page.html new file mode 100644 index 0000000..32d6582 --- /dev/null +++ b/_layouts/page.html @@ -0,0 +1,20 @@ +--- +layout: default +--- + +{% include lang.html %} + +
+ {% if page.layout == 'page' or page.collection == 'tabs' %} + {% assign tab_key = page.title | downcase %} + {% assign title = site.data.locales[lang].tabs[tab_key] | default: page.title %} +

+ {{ title }} +

+
+ {{ content }} +
+ {% else %} + {{ content }} + {% endif %} +
diff --git a/_layouts/post.html b/_layouts/post.html new file mode 100644 index 0000000..f17ceea --- /dev/null +++ b/_layouts/post.html @@ -0,0 +1,152 @@ +--- +layout: default +refactor: true +panel_includes: + - toc +tail_includes: + - related-posts + - post-nav + - comments +--- + +{% include lang.html %} + +
+
+

{{ page.title }}

+ {% if page.description %} +

{{ page.description }}

+ {% endif %} + + +
+ +
+ {{ content }} +
+ +
+ + {% if page.categories.size > 0 %} + + {% endif %} + + + {% if page.tags.size > 0 %} + + {% endif %} + +
+
+ {% if site.data.locales[lang].copyright.license.template %} + {% capture _replacement %} + + {{ site.data.locales[lang].copyright.license.name }} + + {% endcapture %} + + {{ site.data.locales[lang].copyright.license.template | replace: ':LICENSE_NAME', _replacement }} + {% endif %} +
+ + {% include post-sharing.html lang=lang %} +
+ +
+ +
diff --git a/_layouts/tag.html b/_layouts/tag.html new file mode 100644 index 0000000..d766d09 --- /dev/null +++ b/_layouts/tag.html @@ -0,0 +1,23 @@ +--- +layout: page +# The layout for Tag page +--- + +{% include lang.html %} + +
+

+ + {{ page.title }} + {{ page.posts | size }} +

+
    + {% for post in page.posts %} +
  • + {{ post.title }} + + {% include datetime.html date=post.date class='text-muted small text-nowrap' lang=lang %} +
  • + {% endfor %} +
+
diff --git a/_layouts/tags.html b/_layouts/tags.html new file mode 100644 index 0000000..7800ca0 --- /dev/null +++ b/_layouts/tags.html @@ -0,0 +1,22 @@ +--- +layout: page +# All the Tags of posts. +--- + +
+ {% assign tags = '' | split: '' %} + {% for t in site.tags %} + {% assign tags = tags | push: t[0] %} + {% endfor %} + + {% assign sorted_tags = tags | sort_natural %} + + {% for t in sorted_tags %} + + {% endfor %} +
diff --git a/_posts/2024-08-11-first.md b/_posts/2024-08-11-first.md new file mode 100644 index 0000000..c0e4384 --- /dev/null +++ b/_posts/2024-08-11-first.md @@ -0,0 +1,11 @@ +--- +title: Common Linux Commands +author: zjpzhao +date: 2021-05-09 20:19:00 +0800 +categories: [Linux] +tags: [Ubuntu,Shell,TODO] +--- + +## Understand a Command +1. `$ man command` for manual pages +2. `$ command --help` for help diff --git "a/_posts/coding/2024-08-13-Python\350\216\267\345\217\226\347\261\273\347\232\204\345\261\236\346\200\247.md" "b/_posts/coding/2024-08-13-Python\350\216\267\345\217\226\347\261\273\347\232\204\345\261\236\346\200\247.md" new file mode 100644 index 0000000..89d933d --- /dev/null +++ "b/_posts/coding/2024-08-13-Python\350\216\267\345\217\226\347\261\273\347\232\204\345\261\236\346\200\247.md" @@ -0,0 +1,98 @@ +--- +title: Python获取类的属性 +author: X +date: 2024-08-13 09:06:31 +0800 +categories: [coding] +tags: [数据处理] +--- + +# Python 获取类的属性 + +在本文中,我们将介绍如何获取 Python 中类的属性。 + +## 介绍属性 + +在 Python 中,类是通过属性和方法来定义的。属性是类的特征,方法是类的行为。了解如何获取类的属性是非常重要的,因为它允许我们在运行时动态地访问和操作类的特征。 + +## 使用 dir() 函数获取属性 + +Python 内置的`dir()`函数可以获取一个对象的所有属性和方法。当我们调用`dir()`函数时,它会返回一个包含字符串的列表,每个字符串表示对象的一个属性或方法。我们可以通过这个列表来查看类的所有属性。 + +下面是一个示例,演示如何使用`dir()`函数来获取类的属性: + +```python +class Person: + def __init__(self, name, age): + self.name = name + self.age = age + +person = Person("Alice", 25) + +print(dir(person)) +``` + +输出结果: + +```python +['__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', '__weakref__', 'age', 'name'] +``` + +从输出结果中,我们可以看到类的属性`name`和`age`被列出。 + +## 使用 vars() 函数获取属性 + +除了`dir()`函数之外,我们还可以使用内置的`vars()`函数来获取类的属性。`vars()`函数返回一个字典,其中键是属性名,值是属性的值。这个函数只适用于类的实例,而不是类本身。 + +下面是一个示例,演示如何使用`vars()`函数来获取类的属性: + +```python +class Circle: + def __init__(self, radius): + self.radius = radius + +circle = Circle(5) + +print(vars(circle)) +``` + +输出结果: + +```python +{'radius': 5} +``` + +从输出结果中,我们可以看到类的属性`radius`及其对应的值被列出。 + +## 使用 getattr() 和 hasattr() 函数 + +除了`dir()`和`vars()`函数之外,我们还可以使用内置的`getattr()`和`hasattr()`函数来获取类的属性。`getattr()`函数用于获取属性的值,`hasattr()`函数用于检查属性是否存在。 + +下面是一个示例,演示如何使用`getattr()`和`hasattr()`函数来获取和检查类的属性: + +```python +class Car: + def __init__(self, brand, model, year): + self.brand = brand + self.model = model + self.year = year + +car = Car("Tesla", "Model S", 2022) + +model = getattr(car, "model") +print(model) + +if hasattr(car, "color"): + color = getattr(car, "color") + print(color) +else: + print("The car has no color.") +``` + +输出结果: + +```python +Model S +The car has no color. +``` + +在上面的示例中,我们首先使用`getattr()`函数获取类的属性`model`的值,并打印出来。然后使用`hasattr()`函数检查类的属性`color`是否存在,如果存在则使用`getattr()`函数获取其值,并打印出来。如果属性`color`不存在,则输出一个提示消息。 diff --git "a/_posts/coding/2024-08-13-chatglm-PPO\350\256\255\347\273\203\350\267\257\345\276\204\346\216\242\347\264\242.md" "b/_posts/coding/2024-08-13-chatglm-PPO\350\256\255\347\273\203\350\267\257\345\276\204\346\216\242\347\264\242.md" new file mode 100644 index 0000000..e9b7efc --- /dev/null +++ "b/_posts/coding/2024-08-13-chatglm-PPO\350\256\255\347\273\203\350\267\257\345\276\204\346\216\242\347\264\242.md" @@ -0,0 +1,95 @@ +--- +title: chatglm-PPO训练路径探索 +author: X +date: 2024-08-13 09:06:31 +0800 +categories: [coding] +tags: [ppo,模型训练] +--- +# 背景 + +已有基于求解器的评测系统,等于在数学建模大模型这一品类下,不存在人类专家奖励分数难以获取的情况,可以直接获取标准化的评测分数。因此,初步调研认为,使用PPO无疑是方便且能够最大程度发挥数学建模大模型特色的。 + +# RLHF调研 + +## 参考资料 + +[ChatGPT 背后的“功臣”——RLHF 技术详解 (huggingface.co)](https://huggingface.co/blog/zh/rlhf) + +步骤1:收集数据与有监督训练策略 + +从数据集中采样的prompt提问 + +数据标注者(人工)给出最合理的回答,组成问答机制对< Q , A > + +利用问答机制通过SFT有监督精调GPT3.5,得到策略policy + +图片 + +步骤2:收集数据训练奖励模型 + +继续采样prompt,将prompt输入一个或多个 LLM 生成对比数据。他们产生了几对提示-答案< Q , A > + +人类标注者根据模型回答的质量,对回答的好坏进行排序(收集人类反馈) + +得到排序的数据集后,训练奖励模型,奖励模型能够根据输入给出一个标量奖励值,代表人类对这些输出或行为的偏好 +经过充分的训练,奖励模型可以在没有人为干预的情况下对智能体的输出或行为进行打分,以量化其符合人类偏好的程度 + +图片 + +图中的ELO是指建立其人类对于输出的相对排名 + +步骤3:结合奖励模型利用强化学习算法如PPO算法来优化策略 + +再次采样prompt,利用PPO模型(由前面第一步得到的策略初始化)产生结果得到< Q , A > + +将< Q , A > 输入奖励模型,产生打分(奖励) + +利用奖励信号评估策略的输出,通过强化学习算法来优化策略(比如 PPO模型) + +创建一个循环来优化微调策略:通过新采样的数据,在强化学习过程中,策略会生成新的输出或行为,并根据奖励模型的反馈进行迭代优化。这个过程会不断重复,直到模型的性能达到满意的水平 + +图片 + +**容易发现,对于当前任务,我们可以梳理出一个新的步骤,其重点在于把步骤2的奖励模型替换成现有的评测系统。** + +**这显然是可以等价的,细节区别是,由于奖励模型的评分基于人类,因此直接打分会导致标准不统一,所以最终使用的分数实际上是不同LM的排名结果进行归一化。但这一点,基于图距离和求解器的建模评测系统不需要在乎,因为系统计算的分数本就是标准统一的(本就是归一化的)。** + +*认为已经对大框架较为清晰,接下来就需要聚焦于步骤3,以此来看输入输出要求及精细的架构究竟如何。* + +## 步骤三调研 + +前略 + +老弟我真服了,玩我是吧 + +``` +File ~/.cache/huggingface/modules/transformers_modules/chatglm3-6b/modeling_chatglm.py:413, in SelfAttention.forward(self, hidden_states, attention_mask, rotary_pos_emb, kv_cache, use_cache) + 411 # adjust key and value for inference + 412 if kv_cache is not None: +--> 413 cache_k, cache_v = kv_cache + 414 key_layer = torch.cat((cache_k, key_layer), dim=0) + 415 value_layer = torch.cat((cache_v, value_layer), dim=0) + +ValueError: too many values to unpack (expected 2) +``` + +屡次测试glm训练都报类似的错,左右调试(下了llama3和gpt2等各种模型)都能跑通,就glm不行,我说咋了呢,对齐都对到模型层级别了: + +```python +model = AutoModel.from_pretrained(model_name) +model.lm_head = model.transformer.output_layer +del model.transformer.output_layer +model = AutoModelForCausalLMWithValueHead.from_pretrained(model) +``` + +*以上是glm没有lm_head层,本人精心排查后确定output_layer和lm_head同功能,因此进行了层名替换* + +终于是把需要的加载上了,结果还是不行, + +24.07.18 问题解决 + +一开始还以为是glm下错了(事实证明这种思路是错误的,但也算和真相沾边) +使用glm4做其他任务时仍旧有类似情况发生,因此怀疑是包冲突或glm本身的问题,果然: + +[glm-4-9b-chat-1m加载推理汇报错 · Issue #263 · THUDM/GLM-4 · GitHub](https://github.com/THUDM/GLM-4/issues/263) +测验`pip install transformers==4.40`后,成功运行。 diff --git "a/_posts/coding/2024-08-13-conda\347\216\257\345\242\203\345\272\223\344\277\241\346\201\257\345\257\274\345\207\272\344\270\216\345\244\215\345\216\237.md" "b/_posts/coding/2024-08-13-conda\347\216\257\345\242\203\345\272\223\344\277\241\346\201\257\345\257\274\345\207\272\344\270\216\345\244\215\345\216\237.md" new file mode 100644 index 0000000..a8e557a --- /dev/null +++ "b/_posts/coding/2024-08-13-conda\347\216\257\345\242\203\345\272\223\344\277\241\346\201\257\345\257\274\345\207\272\344\270\216\345\244\215\345\216\237.md" @@ -0,0 +1,43 @@ +--- +title: conda环境库信息导出与复原 +author: X +date: 2024-08-13 09:06:31 +0800 +categories: [engineering] +tags: [environment] +--- + +# 背景 + +跑评测,跑着跑着conda环境被人改了,直接跑崩,导师让我把环境保存一下,以便以后遇到这种情况可以复原。 + +下面介绍如何在Conda环境中导出当前所有安装的库(包)的信息,以便在将来某个时刻可以恢复或重建这个环境。 + +# 步骤 + +Conda是一个流行的包管理器和环境管理器,广泛用于Python和其他语言的科学计算。 + +要导出环境的库信息并保存,你可以按照以下步骤操作: + +1. 打开命令行界面(在Windows上是CMD或PowerShell,在macOS或Linux上是Terminal)。 +2. 激活你想要导出的Conda环境。如果你的环境名为`myenv`,你可以使用以下命令: + + ```sh + conda activate myenv + ``` +3. 使用`conda list`命令列出环境中所有已安装的包及其版本信息。这将显示一个列表,包括每个包的名称和版本。 +4. 导出环境的详细信息。你可以使用`conda env export`命令将环境的详细信息导出到一个YAML文件中。例如: + + ```sh + conda env export > environment.yml + ``` + + 这将创建一个名为`environment.yml`的文件,其中包含了所有包的列表和版本信息,以及环境的其他配置。 +5. 将这个`environment.yml`文件保存在安全的地方。当你需要重建环境时,你可以使用以下命令: + + ```sh + conda env create -f environment.yml + ``` + + 这将根据`environment.yml`文件中列出的包和版本信息创建一个新的Conda环境。 + +通过这种方式,你可以确保你的开发环境的一致性和可重复性,无论在何时何地都可以快速地恢复或重建你的工作环境。 diff --git "a/_posts/coding/2024-08-13-json.dumps\350\276\223\345\207\272\347\276\216\345\214\226\347\211\210json.md" "b/_posts/coding/2024-08-13-json.dumps\350\276\223\345\207\272\347\276\216\345\214\226\347\211\210json.md" new file mode 100644 index 0000000..9767853 --- /dev/null +++ "b/_posts/coding/2024-08-13-json.dumps\350\276\223\345\207\272\347\276\216\345\214\226\347\211\210json.md" @@ -0,0 +1,34 @@ +--- +title: json.dumps输出美化版json +author: X +date: 2024-06-27 15:06:31 +0800 +categories: [coding] +tags: [数据处理] +--- + +你是否还在为json.dumps输出的json-string太丑而悲伤? +你是否倦怠于每次利用ide格式化手动美化结果? +一个入参解决你的问题! + +```python +json.dumps(self, indent=4) +``` + +另外一个入参可以处理非dict格式的变量,使其也能转化成json-string。 + +```python +json.dumps(self, default=lambda o: o.__dict__, indent=4) +``` + +使用案例: + +```python +class DataSource: + def __init__(self, instruction, output, dataset_id): + self.instruction = instruction + self.output = output + self.dataset_id = dataset_id + + def to_json(self): + return json.dumps(self, default=lambda o: o.__dict__, indent=4) +``` diff --git "a/_posts/coding/2024-08-13-python\345\274\200\345\244\232\350\277\233\347\250\213.md" "b/_posts/coding/2024-08-13-python\345\274\200\345\244\232\350\277\233\347\250\213.md" new file mode 100644 index 0000000..cb4835c --- /dev/null +++ "b/_posts/coding/2024-08-13-python\345\274\200\345\244\232\350\277\233\347\250\213.md" @@ -0,0 +1,77 @@ +--- +title: python开多进程 +author: X +date: 2024-08-13 09:06:31 +0800 +categories: [coding] +tags: [多进程] +--- + +# 多进程一键开启 + +## 使用 `multiprocessing` 库 + +### 经典方法 + +```python +import os +import time +from multiprocessing import Pool + +def long_time_task(name): + print(f'Run Task {name} {os.getpid()}...') + start = time.time() + time.sleep(1) + end = time.time() + print(f'Task {name} Run {(end - start):.2f} seconds.') + return name + +if __name__ == '__main__': + t_one = time.time() + print(f'Parent process {os.getpid()}.') + p = Pool(34) + for i in range(34): + p.apply_async(long_time_task, args=(i,)) + print('waiting for all subprocesses done...') + p.close() + p.join() + t_two = time.time() + print(f'All subprocesses done. {t_two - t_one:.2f} seconds.') +``` + +### 进程池的进程数选用 + +- 可以填入 `multiprocessing.cpu_count()`。 +- IO密集型选用 `multiprocessing.cpu_count() + n`,`n` 根据情况自选。 +- CPU密集型可直接选用 `multiprocessing.cpu_count()`。 + +## 使用 `tqdm` 库 + +### 更简便的方法 + +```python +from tqdm.contrib.concurrent import process_map +import os +import time + +def long_time_task(name, sign): + print(f'Run Task {name} {sign} ({os.getpid()})...') + start = time.time() + time.sleep(1) + end = time.time() + print(f'Task {name} Run {(end - start):.2f} seconds.') + return name + +if __name__ == '__main__': + def _generator(m): + for i in range(m): + yield "my love" + + r = process_map(long_time_task, range(112), _generator(112), max_workers=34) + print(multiprocessing.cpu_count()) + print(r) +``` + +### 特点 + +- 写法更简便,具有进度条功能。 +- 入参需要将参数生成写在内部(传生成器进去)。 diff --git "a/_posts/coding/2024-08-13-\346\225\260\346\215\256\347\273\223\346\236\204\347\237\245\350\257\206\346\261\240.md" "b/_posts/coding/2024-08-13-\346\225\260\346\215\256\347\273\223\346\236\204\347\237\245\350\257\206\346\261\240.md" new file mode 100644 index 0000000..8a6a45f --- /dev/null +++ "b/_posts/coding/2024-08-13-\346\225\260\346\215\256\347\273\223\346\236\204\347\237\245\350\257\206\346\261\240.md" @@ -0,0 +1,62 @@ +--- +title: 数据结构知识池 +author: X +date: 2024-06-27 15:06:31 +0800 +categories: [coding] +tags: [数据结构.数据处理] +--- + +# 批量处理迭代器(列表、元组) + +在Python中,`map()` 函数用于将一个函数应用于迭代器中的每个元素,并返回一个包含结果的新的可迭代对象。它的基本语法如下: + +```python +map(function, iterable) +``` + +其中,`function` 是一个函数对象,`iterable` 是一个可迭代对象,比如列表、元组等。 + +`map()` 函数将 `iterable` 中的每个元素依次传递给 `function` 进行处理,并返回一个包含处理结果的新的可迭代对象。`function` 可以是内置函数、自定义函数或Lambda函数。 + +## 示例 + +### 对列表中的每个元素进行平方操作: + +```python +numbers = [1, 2, 3, 4, 5] +squared_numbers = list(map(lambda x: x**2, numbers)) +print(squared_numbers) # 输出:[1, 4, 9, 16, 25] +``` + +### 将列表中的每个元素转换为字符串: + +```python +numbers = [1, 2, 3, 4, 5] +number_strings = list(map(str, numbers)) +print(number_strings) # 输出:['1', '2', '3', '4', '5'] +``` + +### 使用自定义函数进行处理: + +```python +def double(x): + return 2 * x + +numbers = [1, 2, 3, 4, 5] +doubled_numbers = list(map(double, numbers)) +print(doubled_numbers) # 输出:[2, 4, 6, 8, 10] +``` + +## 注意事项 + +- `map()` 函数返回的是一个迭代器对象,如果需要得到一个列表对象,可以使用 `list()` 函数将其转换为列表。 + +# DF的四舍五入 + +例如,如果你有一个数据框 `df`,其中一列名为 `'col1'`,你想要将 `'col1'` 中的所有数值四舍五入到小数点后三位,你可以使用以下代码: + +```python +df['col1'] = df['col1'].round(3) +``` + +这将更新 `df` 中的 `'col1'` 列,使其所有数值都四舍五入到小数点后三位。 diff --git "a/_posts/coding/2024-08-13-\350\204\232\346\234\254\345\206\205\346\214\207\345\256\232GPU\351\203\250\347\275\262.md" "b/_posts/coding/2024-08-13-\350\204\232\346\234\254\345\206\205\346\214\207\345\256\232GPU\351\203\250\347\275\262.md" new file mode 100644 index 0000000..dc3660f --- /dev/null +++ "b/_posts/coding/2024-08-13-\350\204\232\346\234\254\345\206\205\346\214\207\345\256\232GPU\351\203\250\347\275\262.md" @@ -0,0 +1,61 @@ +--- +title: 脚本内指定GPU部署 +author: X +date: 2024-08-13 09:06:31 +0800 +categories: [engineering] +tags: [GPU,CUDA] +--- + +# CUDA设备配置 + +## 设置脚本可见的GPU卡号 + +在脚本开头添加以下代码,以指定脚本可见的GPU卡号范围: + +```python +import os + +# 设置环境变量,只显示4, 5, 6, 7号GPU +os.environ['CUDA_VISIBLE_DEVICES'] = '4,5,6,7' +``` + +## 注意事项 + +- 经过上述设置后,对于脚本而言,这四张卡的编号将会是 `"0, 1, 2, 3"`,即根据可见卡重新排列了索引。 +- 在此脚本内的任何GPU部署都不会影响到其他卡,因为此脚本看不到其他卡了。 + +## 模型加载示例 + +以下是使用指定设备映射加载模型的示例代码: + +```python +from transformers import AutoModelForCausalLM + +# 模型路径 +model_path = 'path_to_your_model' + +# 加载模型,指定数据类型、内存使用优化、信任远程代码,并自动设置设备映射 +model = AutoModelForCausalLM.from_pretrained( + model_path, + torch_dtype=torch.bfloat16, + low_cpu_mem_usage=True, + trust_remote_code=True, + device_map="auto" +) +``` + +`device_map` 参数可以自由设置,不会影响其他人的卡。 + +## 附赠 +查看GPU进程全名 + +``` +nvidia-smi --query-compute-apps=process\_name --format=csv +``` + +查看GPU信息 + +``` +nvidia-smi +``` + diff --git "a/_posts/engineering/2024-08-13-linux\345\210\240\351\231\244\347\233\270\345\205\263.md" "b/_posts/engineering/2024-08-13-linux\345\210\240\351\231\244\347\233\270\345\205\263.md" new file mode 100644 index 0000000..a9bd4df --- /dev/null +++ "b/_posts/engineering/2024-08-13-linux\345\210\240\351\231\244\347\233\270\345\205\263.md" @@ -0,0 +1,172 @@ +--- +title: linux删除相关 +author: X +date: 2024-08-13 09:06:31 +0800 +categories: [engineering] +tags: [linux] +--- +# 常用 + +在Linux系统中,您可以使用`find`命令结合`rm`命令来删除所有以"temp"开头的`.py`文件。以下是执行这个操作的命令: + +```bash +find ./ -type f -name 'temp*.py' -exec rm -f {} \; +``` + +这个命令的解释如下: + +- `find ./`:从当前目录开始查找文件。 +- `-type f`:只查找文件。 +- `-name 'temp*.py'`:查找文件名以"temp"开头,并且扩展名为".py"的文件。 +- `-exec rm -f {} \;`:对找到的每个文件执行`rm -f`命令,`-f`选项表示强制删除,不提示确认。`{}`是一个占位符,代表`find`命令找到的每个文件的路径。 + +**注意**:这个命令会递归地搜索整个文件系统,并且删除所有匹配的文件。在执行之前,请确保您了解这个命令的作用,并且已经备份了所有重要的数据。如果您只想在特定目录下执行这个操作,可以将`/`替换成您想要搜索的目录路径。 + +# linux中快速删除的几种方法 + +## 一. rm -rf + +对于这个命令相信很多很熟悉,甚至一度说rm -rf /*是跑路命令,然后实时不然,很多需要删除的文件都需要这个命令,当然使用得当看个人,对于文件量不大情况还是比较有效的,具体如下: + +``` +rm -rf <目录或文件> +``` + +其中,-r 参数表示递归删除目录及其内容,-f 参数表示强制删除,忽略不存在的文件或目录和删除的确认提示。请注意,该命令将永久删除文件或目录,无法恢复。 + +以下是几个示例: + +### 1,删除一个文件: + +``` +rm -rf example.txt +``` + +该命令将递归地删除 example.txt 文件。 + +### 2,删除一个目录及其内容: + +``` +rm -rf myDirectory +``` + +该命令将递归地删除 myDirectory 目录及其所有内容,包括子目录和文件。 + +### 3,删除多个文件或目录: + +rm -rf file1 file2 dir1 dir2 + +该命令将递归地删除多个文件和目录。 + +### 4,使用通配符删除一类文件: + +``` +rm -rf *.txt +``` + +该命令将递归地删除当前目录下所有以 .txt 结尾的文件。常用通配符如: + +1,`*`:匹配零个或多个字符,用于匹配任意字符串。 +例如:rm -rf prefix*,可以删除以 “prefix” 开头的所有文件。 + +2,`?`:匹配一个字符,用于匹配单个字符。 +例如:rm -rf file?.txt,可以删除名为 “file1.txt”、“file2.txt” 等的文件。 + +3,`[]`:匹配括号内的任意一个字符。 +例如:rm -rf [abc]*,可以删除以 “a”、“b” 或 “c” 开头的文件。 + +4,`{}`:用逗号分隔的一组项,匹配其中的任意一项。 +例如:rm -rf {file1,file2}.txt,可以删除 “file1.txt” 和 “file2.txt”。 + +### 5,使用循环结构来批量删除多个目录: + +``` +for dir in dir1 dir2 dir3; do rm -rf "$dir"; done +``` + +该命令将递归地删除当前目录下的目录 dir1、dir2 和 dir3。 + +### 6,删除特定目录下的一类文件: + +``` +rm -rf /path/to/directory/*.log +``` + +该命令将递归地删除 /path/to/directory 目录下所有以 .log 结尾的文件。以下是对您提供的文本内容进行格式处理的结果: + +--- + +## 二. 使用 `rm` 或 `rmdir` 删除文件夹及文件 + +### 1. 删除空文件夹: + +```bash +rmdir foldername +``` + +此命令用于删除名为 `foldername` 的空文件夹。如果文件夹不为空,即包含内容或子文件夹,则命令会报错。 + +### 2. 递归删除文件夹及其内容: + +```bash +rm -r foldername +``` + +该命令递归地删除名为 `foldername` 的文件夹及其所有内容,包括子文件夹和文件。执行前请确保这是您的意图。 + +--- + +## 三. 比 `rm -rf` 更高效的删除方法 + +如果 `rm -rf` 命令在删除大量文件时速度较慢,可以尝试以下更高效的方法: + +1. 使用 `find` 命令结合 `-delete` 参数: + + ```bash + find /path/to/folder -type f -delete + ``` + + 此命令仅删除指定路径 `/path/to/folder` 中的文件,不删除文件夹本身,比 `rm -rf` 更高效,因为它避免了递归遍历子文件夹。 +2. 使用 `rsync` 命令删除文件夹: + + ```bash + rsync -a --delete empty_folder/ path/to/folder/ + ``` + + 此方法通过将空文件夹 `empty_folder` 同步到目标文件夹 `path/to/folder` 上,实际上清空了目标文件夹的内容。 + +--- + +## 四. 查询目前存在的文件个数 + +若需查询文件夹中文件和文件夹的数量,可以使用 `ls` 命令结合一些选项和管道操作符。以下是几种查询方式: + +1. 查询文件夹中的文件和文件夹总数: + + ```bash + ls -lA foldername | wc -l + ``` + + 此命令列出 `foldername` 中的所有文件和文件夹(包括隐藏的),并计算总数。 +2. 查询文件夹中的文件个数: + + ```bash + ls -lA foldername | grep -v '^d' | wc -l + ``` + + 此命令列出 `foldername` 中的所有文件,过滤掉文件夹,并计算文件数量。 +3. 查询文件夹中的文件夹个数: + + ```bash + ls -lA foldername | grep '^d' | wc -l + ``` + + 此命令列出 `foldername` 中的所有文件夹,并计算文件夹数量。 + +--- + +请注意,上述命令中的 `foldername`、`/path/to/folder` 和 `empty_folder` 应替换为实际的文件夹路径。 + +**上述是个人根据常用命令结合官网做的总结,欢迎指导,当删除还是要结合自己需要慎用** +———————————————— +参考链接:https://blog.csdn.net/MatChen/article/details/135404591 diff --git "a/_posts/engineering/2024-08-13-linux\346\234\215\345\212\241\345\231\250\347\246\273\347\272\277\345\256\211\350\243\205anaconda.md" "b/_posts/engineering/2024-08-13-linux\346\234\215\345\212\241\345\231\250\347\246\273\347\272\277\345\256\211\350\243\205anaconda.md" new file mode 100644 index 0000000..800eb2a --- /dev/null +++ "b/_posts/engineering/2024-08-13-linux\346\234\215\345\212\241\345\231\250\347\246\273\347\272\277\345\256\211\350\243\205anaconda.md" @@ -0,0 +1,100 @@ +--- +title: linux服务器离线安装anaconda +author: X +date: 2024-08-13 15:06:31 +0800 +categories: [engineering] +tags: [Linux,environment] +--- + +## 步骤1 下载anaconda.sh至本地 + +下载地址:[Index of / (anaconda.com)](https://repo.anaconda.com/archive/) +服务器版本查看:`uname -a` +下载相应版本的sh文件到本地 + +## 步骤2 上传sh文件至指定服务器 + +这个不多说了,xftp自己传一下 + +## 步骤3 静音安装 + +强调静音安装,是因为常规安装指定会需要不断按回车和输入yes认证,用我的办法可以规避这种麻烦。 + +```bash +mkdir -p ~/anaconda3 +bash ~/yourpath/anaconda.sh -b -u -p ~/anaconda3 +rm -rf ~/yourpath/anaconda.sh +``` + +注意这里bash指令结尾的`~/anaconda3`是安装路径,没写好的话会一大坨装错地方,linux没有撤回,很难受。 +不过执行后控制台会打印perfix参数,这个对应的就是安装路径,快速检查一下对不对,如果错了直接ctrl+C停掉也来得及。 +装完删除安装包,删不删见仁见智,不是必须的。 +然后这时候应该还是调不出conda,初始化一下: + +```bash +~/anaconda3/bin/conda init bash +~/anaconda3/bin/conda init zsh +``` + +控制台会提醒你重启会话,你可以重启也可以刷一下`.bashrc`: + +```bash +source ~/.bashrc +``` + +这时候会发现已经在conda里了。 + +## 步骤4 配置内网源 + +如果没有.condarc文件,需要先使用下面命令生成默认配置文件 + +``` +source ~/.bashrc +conda config --set show_channel_urls yes +``` + +*亲测,可以不这样写,直接`vim ./.condarc`然后把下面的内容粘进去也是完全可以的哈~* + +将.condarc中内容修改为 + +##### 黄区 + +``` +# channels: +# - defaults +channel_priority: strict +show_channel_urls: true +channel_alias: http://10.244.173.8:8088/repository/conda-proxy +default_channels: + - main +``` + +也可以试试下面这个 + +``` +channel_alias: http://10.155.196.240:8088/repository/conda-proxy +default_channels: + - main + - r +``` + +##### 绿区 + +``` +# channels: +# - defaults +channel_priority: strict +show_channel_urls: true +channel_alias: http://10.155.97.225:8088/repository/conda-proxy +default_channels: + - main +``` + +一些小技巧是可以先ping一下,比如`ping 10.155.97.225`,如果能通,那一般这个配置就是好用的。ping都ping不通的话,建议在w3找找有没有其他ip。 + +**随后就可以愉快的使用了** + +参考文档: +[linux conda安装、配置源 - 潘志的博客 (huawei.com)](https://3ms.huawei.com/km/blogs/details/13349249) +[新Linux机器食用方法\_pip与conda配置\_pycharm远程连接配置\_yum配置\_gcc安装与升级 - AI特性与推理服务部 - 3MS知识管理社区 (huawei.com)](https://3ms.huawei.com/km/groups/3773717/blogs/details/15307199) +一些操作借鉴的miniconda的官方文档:[Miniconda — Anaconda documentation](https://docs.anaconda.com/miniconda/) diff --git "a/_posts/engineering/2024-08-13-linux\350\275\257\351\223\276\346\216\245\347\233\270\345\205\263.md" "b/_posts/engineering/2024-08-13-linux\350\275\257\351\223\276\346\216\245\347\233\270\345\205\263.md" new file mode 100644 index 0000000..d981d26 --- /dev/null +++ "b/_posts/engineering/2024-08-13-linux\350\275\257\351\223\276\346\216\245\347\233\270\345\205\263.md" @@ -0,0 +1,69 @@ +--- +title: linux软链接相关|含批量脚本 +author: X +date: 2024-08-13 09:06:31 +0800 +categories: [engineering] +tags: [黑科技,linux] +--- + +## 创建软链接 + +```bash +ln -s [源文件或目录] [目标文件或目录] +``` + +## 修改软链接 + +```bash +ln --snf [新的源文件或目录] [目标文件或目录] +``` + +## 删除软链接 + +```bash +rm -rf 软链接名称 +``` + +或 + +```bash +rm 软链接名称 # 不加 -rf 也可以 +``` + +## 查看帮助 + +```bash +ln --help +``` + +## 批量更新软链接源文件路径 + +```bash +#!/bin/bash + +# 定义旧的和新的源文件路径前缀 +old_path_prefix="/123/" +new_path_prefix="/456/" + +# 遍历当前目录下的所有软链接 +for symlink in $(find . -type l); do + # 使用 readlink 命令获取软链接的绝对路径 + target=$(readlink -e "$symlink") + + # 检查目标路径是否以旧路径前缀开头 + if [[ "$target" == "$old_path_prefix"* ]]; then + # 构造新的软链接路径 + new_target="${target/$old_path_prefix/$new_path_prefix}" + + # 输出将要进行的操作 + echo "Updating symlink $symlink to point to $new_target" + + # 更新软链接的源文件路径 + ln -sfn "$new_target" "$symlink" + fi +done + +echo "Symlink update completed." +``` + +将此脚本保存为 `.sh` 文件,然后拖放到目标目录中运行。 diff --git "a/_posts/engineering/2024-08-13-python\347\210\254\350\231\253\345\233\276\350\247\243" "b/_posts/engineering/2024-08-13-python\347\210\254\350\231\253\345\233\276\350\247\243" new file mode 100644 index 0000000..4d35e2c --- /dev/null +++ "b/_posts/engineering/2024-08-13-python\347\210\254\350\231\253\345\233\276\350\247\243" @@ -0,0 +1,110 @@ +--- +title: python爬虫图解 +author: X +date: 2024-08-13 09:06:31 +0800 +categories: [engineering] +tags: [黑科技,爬虫] +--- + +# 发送http请求 + +以csdn为例,鼠标右键点击检查(或`F12`进入开发者模式),然后点击Network,刷新网页,继续点击Name列表中的第一个。我们发现此网站的请求方式为GET,请求头Headers反映用户电脑系统、浏览器版本等信息。 + +! [image]() + +! [image]() + +在展开框中选择请求标头,一般拿到Cookie和User-Agent就可以(前者用来加载登陆状态,后者用来伪装浏览器访问)。 + +构造一个请求 + +``` +url = 'https://blog.csdn.net/xxx/.html' +header = { + "User-Agent": "Mozilla/5.0 (xxx)", + "Cookie": "uuid_tt_dd=xxxxx" +} +response = requests.get(url, headers=header, verify=False, timeout=10) +``` + +`verify`设为False可以规避认证问题,同样也存在风险(有可能被第三方攻击)。真正的勇士从不在意这些。 + +作为进阶,你还可以随机设一些user-agent列表来模拟不同浏览器访问。 + +# 解析网页内容 + +有很多方式,这里使用` BeautifulSoup`(TODO: 不同解析库的异同、哪个是如今最流行的) + +``` +soup = BeautifulSoup(response.content, 'html.parser') +print(soup.prettify()) # 输出页面内容,帮助你理解页面结构 +# 提取文章链接 +article_list = soup.find('ul', class_='column_article_list') +``` + +基本上,soup.find能解决百分之90的问题,核心在于你要先找到自己想爬取的内容在哪里。 +一种方式是,在你想要爬取的内容上右击,选择`检查`,会自动引到对应的源代码。 + +图片区域 + +以上是通过右击找到了csdn首页的推荐文章列表,如果想要爬取首页这一列所有的推荐文章链接,可以看到,它们都属于`
`这个大类。那么你可以: + +```python +article_list = soup.find('div', class_='Community') # 获取文章列表(小类列表) +``` + +实际内容往往层层嵌套,拿到所有小类并不是终极: + +图片区域 + +上图红框里的`a`中的`href`字段才是你想要的链接。那么你需要找到所有小类列表中的`
`,再找到`a`,再去获取`a`中的`href`: + +``` +if not article_list: + print("No article list found.") + +else: + article_links = [] + article_items = article_list.find_all('div', class_='content') + for item in article_items: + a_tag = item.find('a') + if a_tag and 'href' in a_tag.attrs: + link = a_tag['href'] + article_links.append(link) +``` + +这样就拿到了。后面可以再读取文章内容、标题,然后保存为md什么的,原理是一样的: + +```python +from markdownify import markdownify as md + +# 遍历文章链接,获取文章内容并转换为Markdown格式 +for link in article_links: + article_response = requests.get(link, headers=header, verify=False, timeout=10) + article_soup = BeautifulSoup(article_response.content, 'html.parser') + + # 提取文章内容 + article_content = article_soup.find('div', id='content_views') + if article_content: + # 转换为Markdown格式 + markdown_content = md(str(article_content)) + + # 获取文章标题 + title = article_soup.find('h1', class_='title-article').text.strip().replace(" ", "").replace("/", "-") + + # 保存为Markdown文件 + filename = f"{output_dir}/{title}.md" + with open(filename, 'w', encoding='utf-8') as f: + f.write(markdown_content) + print(f"Saved {filename}") + else: + print(f"No content found for {link}") + wait_time = random.uniform(1, 10) # 模拟人类操作 + time.sleep(wait_time) + +print("All articles have been processed.") +``` + +一些需要注意的小tips: + +`random.uniform`: 随机生成一个在[x,y]范围内的实数 diff --git "a/_posts/engineering/2024-08-13-transformers.Trainer\344\270\255wandb\347\232\204\345\205\263\351\227\255.md" "b/_posts/engineering/2024-08-13-transformers.Trainer\344\270\255wandb\347\232\204\345\205\263\351\227\255.md" new file mode 100644 index 0000000..30146ed --- /dev/null +++ "b/_posts/engineering/2024-08-13-transformers.Trainer\344\270\255wandb\347\232\204\345\205\263\351\227\255.md" @@ -0,0 +1,65 @@ +--- +title: transformers.Trainer中wandb的关闭 +author: X +date: 2024-08-13 09:06:31 +0800 +categories: [engineering] +tags: [黑科技,模型训练,transformers] +--- + +# wandb关闭 + +## 背景 + +``` +[INFO|integration_utils.py:750] 2024-07-08 16:22:45,186 >> Automatic Weights & Biases logging enabled, to disable set os.environ["WANDB_DISABLED"] = "true" +wandb: WARNING The `run_name` is currently set to the same value as `TrainingArguments.output_dir`. If this was not intended, please specify a different run name by setting the `TrainingArguments.run_name` parameter. +wandb: Network error (SSLError), entering retry loop. +wandb: W&B API key is configured. Use `wandb login --relogin` to force relogin +wandb: Network error (SSLError), entering retry loop. +wandb: ERROR Run initialization has timed out after 90.0 sec. +wandb: ERROR Please refer to the documentation for additional information: https://docs.wandb.ai/guides/track/tracking-faq#initstarterror-error-communicating-with-wandb-process- +``` + +使用transformers.Trainer过程中,wandb连不上,想着直接关了算了。 + +## 1. 在Python代码中设置路径 + +在代码最前面写: + +```python +import os +os.environ["WANDB_DISABLED"]="true" +``` + +这样做的缺点是会报警告信息: + +``` +Using the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging results (for instance --report_to none). +``` + +另外,使用模型训练pipeline时,一般也不会想要进到已成型的脚本里再做改动,这种思路总感觉怪怪的,缺乏可拓展性和稳定性。 + +## 2. 在TrainingArguments中设置入参 + +在TrainingArguments的入参中设置`report_to="none"`(事实上这也是第一种方法警告信息中推荐的方案)。 + +``` +report_to: none +``` + +实践中往往在yaml文件中找到train参数的位置,贴上去即可。 +此处还尝试过改成`report_to: tensorboard`,结果tensorboard也没打开: + +``` +File "/home/ma-user/anaconda3/envs/chatglm/lib/python3.10/site-packages/transformers/trainer_callback.py", line 421, in add_callback + cb = callback() if isinstance(callback, type) else callback + File "/home/ma-user/anaconda3/envs/chatglm/lib/python3.10/site-packages/transformers/integrations/integration_utils.py", line 603, in __init__ + raise RuntimeError( +RuntimeError: TensorBoardCallback requires tensorboard to be installed. Either update your PyTorch version or install tensorboardX. +``` + +直接`none`解千愁得了。 + +## 参考资料 + +- [How to turn WanDB off in trainer? - #9 by mspinaci - Beginners - Hugging Face Forums](https://discuss.huggingface.co/t/how-to-turn-wandb-off-in-trainer/6237/9) diff --git "a/_posts/engineering/2024-08-13-\344\270\244\345\217\260Linux\346\234\272\345\231\250\344\274\240\346\226\207\344\273\266.md" "b/_posts/engineering/2024-08-13-\344\270\244\345\217\260Linux\346\234\272\345\231\250\344\274\240\346\226\207\344\273\266.md" new file mode 100644 index 0000000..0944812 --- /dev/null +++ "b/_posts/engineering/2024-08-13-\344\270\244\345\217\260Linux\346\234\272\345\231\250\344\274\240\346\226\207\344\273\266.md" @@ -0,0 +1,47 @@ +--- +title: 两台Linux机器传文件 +author: X +date: 2024-08-13 09:06:31 +0800 +categories: [engineering] +tags: [文件传输,linux] +--- +本来用的是xftp里直接传,结果速度很慢而且直接给哥们C盘干爆了。 +难道这种方式传输也是要走本地的?不然为什么C盘会爆? +取消任务之后就好了,说明就是传输过程经过了本地的C盘。 +所以还是去找别的办法,依稀记得有个scp方法: + +scp +【优点】简单方便,安全可靠;支持限速参数 + +【缺点】不支持排除目录 + +【用法】 + +scp就是secure copy,是用来进行远程文件拷贝的。数据传输使用 ssh,并且和ssh 使用相同的认证方式,提供相同的安全保证 。 + +命令格式: + +scp [参数] <源地址(用户名@IP地址或主机名)>:<文件路径> <目的地址(用户名 @IP 地址或主机名)>:<文件路径> + +举例: + +scp /home/work/source.txt work@192.168.0.10:/home/work/ #把本地的source.txt文件拷贝到192.168.0.10机器上的/home/work目录下 + +scp work@192.168.0.10:/home/work/source.txt /home/work/ #把192.168.0.10机器上的source.txt文件拷贝到本地的/home/work目录下 + +scp work@192.168.0.10:/home/work/source.txt work@192.168.0.11:/home/work/ #把192.168.0.10机器上的source.txt文件拷贝到192.168.0.11机器的/home/work目录下 + +scp -r /home/work/sourcedir work@192.168.0.10:/home/work/ #拷贝文件夹,加-r参数 + +scp -r /home/work/sourcedir work@www.myhost.com:/home/work/ #使用主机名 + +scp -r -v /home/work/sourcedir work@www.myhost.com:/home/work/ #显示详情,加-v参数 +———————————————— + +原文链接:https://blog.csdn.net/FL1768317420/article/details/136989778 + +但当时看到这个方法后,觉得没看到做认证的地方啊? +所以一直没用,首先用了设好认证的xftp。 +现在走投无路,用了一个日志文件传了一下试试,结果控制台是会出现一个让你输入password的部分,输入回车后就直接开始传输了,非常便捷,而且速度很快。 + +注意事项:不用写端口,写了反而报错。直接work@192.168.0.11:/home/work/,冒号后面直接跟目标文件夹即可。 diff --git "a/_posts/game/2024-06-27-\347\214\253\345\233\275\345\273\272\350\256\276\350\200\205\345\216\237\347\211\210\345\274\200\345\220\257\345\260\217\347\214\253\347\247\221\345\255\246\345\256\266.md" "b/_posts/game/2024-06-27-\347\214\253\345\233\275\345\273\272\350\256\276\350\200\205\345\216\237\347\211\210\345\274\200\345\220\257\345\260\217\347\214\253\347\247\221\345\255\246\345\256\266.md" new file mode 100644 index 0000000..c83a445 --- /dev/null +++ "b/_posts/game/2024-06-27-\347\214\253\345\233\275\345\273\272\350\256\276\350\200\205\345\216\237\347\211\210\345\274\200\345\220\257\345\260\217\347\214\253\347\247\221\345\255\246\345\256\266.md" @@ -0,0 +1,32 @@ +--- +title: 猫国建设者原版开启小猫科学家 +author: X +date: 2024-06-27 15:06:31 +0800 +categories: [game] +tags: [猫国建设者,黑科技] +--- + +原版地址: + +Kittens Game - a Dark Souls of incremental gaming + +原版是有中文的。 + +配套的小猫科学家github地址: + +https://github.com/kitten-science/kitten-scientists + +这里面提供了插件和小书签,可以选一个装,我们装小书签就可以了,比较简单。 + +小书签: + +``` +javascript:(function(){var d=document,s=d.createElement('script');s.src='https://kitten-science.com/stable.js';d.body.appendChild(s);})(); +``` + +复制之后放到收藏夹里(edge直接在收藏夹栏右键粘贴即可,其他的改url就行) + +然后打开原版地址,点一下小书签,等几秒就加载上了,右上角: + +![0](assets/img/62b2e090f52143f486eb80aaaad87944.png) + diff --git "a/_posts/math/2024-08-13-\346\263\260\345\213\222\345\205\254\345\274\217.md" "b/_posts/math/2024-08-13-\346\263\260\345\213\222\345\205\254\345\274\217.md" new file mode 100644 index 0000000..245c66e --- /dev/null +++ "b/_posts/math/2024-08-13-\346\263\260\345\213\222\345\205\254\345\274\217.md" @@ -0,0 +1,62 @@ +--- +title: 泰勒公式 +author: X +date: 2024-08-13 09:06:31 +0800 +categories: [math] +tags: [自然哲学的数学原理,latex] +--- + +泰勒公式是数学分析中的一个重要工具,它用于将一个复杂函数在某一点附近用多项式逼近。这个公式由英国数学家布鲁克·泰勒(Brook Taylor)在1715年提出。泰勒公式的基本思想是,如果一个函数在某一点处足够光滑(即具有足够高的连续导数),那么该函数在该点附近可以用一个多项式来近似表示。 + +### 泰勒公式的形式 + +假设函数 $ f(x) $ 在点 $ a $ 处有 $ n+1 $ 阶连续导数,那么对于 $ x $ 在 $ a $ 附近的值,函数 $ f(x) $ 可以表示为: + +$$ +f(x) = f(a) + f'(a)(x-a) + \frac{f''(a)}{2!}(x-a)^2 + \cdots + \frac{f^{(n)}(a)}{n!}(x-a)^n + R_n(x) +$$ + +其中,$ f^{(n)}(a) $ 表示函数 $ f(x) $ 在点 $ a $ 处的第 $ n $ 阶导数,$ R_n(x) $ 是余项,表示多项式逼近的误差。 + +### 余项 $ R_n(x) $ + +余项 $ R_n(x) $ 有多种形式,常见的两种是拉格朗日余项和皮亚诺余项: + +1. **拉格朗日余项**: + + $$ + R_n(x) = \frac{f^{(n+1)}(\xi)}{(n+1)!}(x-a)^{n+1} + $$ + + 其中 $ \xi $ 是介于 $ a $ 和 $ x $ 之间的一个数。 +2. **皮亚诺余项**: + + $$ + R_n(x) = o((x-a)^n) + $$ + + 表示当 $ x $ 趋近于 $ a $ 时,余项 $ R_n(x) $ 是比 $ (x-a)^n $ 更高阶的无穷小。 + +### 泰勒公式的应用 + +泰勒公式在数学和物理学中有广泛的应用,例如: + +- **函数逼近**:通过泰勒公式,可以用多项式近似复杂函数,简化计算。 +- **数值分析**:在数值计算中,泰勒公式用于估计误差和提高计算精度。 +- **微分方程**:在求解微分方程时,泰勒公式可以用于展开解函数,便于分析和计算。 + +### 示例 + +考虑函数 $ f(x) = e^x $ 在 $ x = 0 $ 处的泰勒展开: + +$$ +e^x = 1 + x + \frac{x^2}{2!} + \frac{x^3}{3!} + \cdots + \frac{x^n}{n!} + R_n(x) +$$ + +其中,$ f^{(n)}(0) = e^0 = 1 $ 对于所有 $ n $,因此: + +$$ +e^x = 1 + x + \frac{x^2}{2} + \frac{x^3}{6} + \cdots + \frac{x^n}{n!} + R_n(x) +$$ + +这个展开式在数值计算中非常有用,特别是在计算 $ e^x $ 的近似值时。 diff --git a/_posts/pool/2024-08-13-generator.md b/_posts/pool/2024-08-13-generator.md new file mode 100644 index 0000000..1aa537c --- /dev/null +++ b/_posts/pool/2024-08-13-generator.md @@ -0,0 +1,50 @@ +--- +title: generator +author: X +date: 2024-08-13 09:06:31 +0800 +categories: [pool] +tags: [] +--- + +要将这些生成器对象合并成一个包含 `n` 行的 JSONL 文件,每行包含字典中的每个字段的一条记录,可以按照以下步骤进行操作: + +1. 从每个生成器中逐条读取数据。 +2. 将每条数据合并成一行。 +3. 将合并后的数据写入 JSONL 文件。 + +以下是一个示例代码,展示了如何实现这一过程: + +```python +import json + +# 假设你的生成器对象存储在一个字典中 +generators = { + 'gpt4': , + 'llama3-8b': , + 'glm4-9b': , + 'qwen2-7b': , +} + +# 打开一个文件用于写入 JSONL 数据 +with open('merged_output.jsonl', 'w') as outfile: + # 逐条读取生成器中的数据 + for _ in range(n): # 假设 n 是每个生成器的条数 + merged_line = {} + for key, generator in generators.items(): + merged_line[key] = next(generator) + + # 将合并后的数据写入 JSONL 文件 + json.dump(merged_line, outfile) + outfile.write('\n') +``` + +在这个示例中: + +1. `generators` 字典包含了你的生成器对象。 +2. `with open('merged_output.jsonl', 'w') as outfile:` 打开一个文件用于写入 JSONL 数据。 +3. `for _ in range(n):` 循环 `n` 次,每次从每个生成器中读取一条数据。 +4. `merged_line = {}` 初始化一个空字典用于存储合并后的数据。 +5. `for key, generator in generators.items():` 遍历每个生成器,读取当前条数据并存储在 `merged_line` 字典中。 +6. `json.dump(merged_line, outfile)` 将合并后的数据写入文件,`outfile.write('\n')` 写入换行符以分隔每行数据。 + +请确保在运行代码之前,将 `n` 替换为实际的生成器条数。这样,你就可以将这些生成器对象合并成一个包含 `n` 行的 JSONL 文件,每行包含字典中的每个字段的一条记录。