Merge pull request #383 from genedna/main

Update the README.md and add an architect image #34

.env

@@ -1,4 +1,4 @@
-# Fillin the following environment variables with values you set
+# Filling the following environment variables with values you set
 ## Logging Configuration
 RUST_LOG = "DEBUG"
 MEGA_LOG_PATH = "/tmp/.mega/logs"
@@ -17,11 +17,11 @@ MEGA_DB_SQLX_LOGGING = false # Whether to disabling SQLx Log
 ## End Database Configuration
 
 
-## SSH/HTTPS Key Configruation
+## SSH/HTTPS Key Configuration
 MEGA_SSH_KEY = "/tmp/.mega/ssh"
 MEGA_HTTPS_PUBLIC_KEY = ""
 MEGA_HTTPS_PRIVATE_KEY = ""
-## End SSH/HTTPS Key Configruation
+## End SSH/HTTPS Key Configuration
 
 ## File Storage Configuration
 MEGA_RAW_STORAGE = "LOCAL" # LOCAL or REMOTE

.env.example

@@ -1,9 +1,10 @@
-# Fillin the following environment variables with values you set
+# Filling the following environment variables with values you set
 ## Logging Configuration
-RUST_LOG = "debug"
+RUST_LOG = "DEBUG"
+MEGA_LOG_PATH = "/tmp/.mega/logs"
 
 ## Database Configuration
-DB = "postgres" # {postgres}
+DB = "" # {postgres}
 DB_USERNAME = ""
 DB_PASSWORD = ""
 DB_HOST = ""
@@ -16,11 +17,11 @@ MEGA_DB_SQLX_LOGGING = false # Whether to disabling SQLx Log
 ## End Database Configuration
 
 
-## SSH/HTTPS Key Configruation
+## SSH/HTTPS Key Configuration
 MEGA_SSH_KEY = "/tmp/.mega/ssh"
 MEGA_HTTPS_PUBLIC_KEY = ""
 MEGA_HTTPS_PRIVATE_KEY = ""
-## End SSH/HTTPS Key Configruation
+## End SSH/HTTPS Key Configuration
 
 ## File Storage Configuration
 MEGA_RAW_STORAGE = "LOCAL" # LOCAL or REMOTE
@@ -39,6 +40,7 @@ MEGA_OBJ_REMOTE_ENDPOINT = "https://obs.cn-east-3.myhuaweicloud.com" # Override
 MEGA_BIG_OBJ_THRESHOLD_SIZE = 1024 # Unit KB.
 
 ## Only import directory support multi-branch commit and tag, repo under regular directory only support main branch only
+## Mega treats files in that directory as import repo and other directories as monorepo
 MEGA_IMPORT_DIRS = "/third-part"
 
 ## Decode cache configuration

.gitignore

@@ -12,42 +12,8 @@ Cargo.lock
 # IDE configurations
 .idea
 .vscode
+.zed
 
 # Mac
 .DS_Store
 .VSCodeCounter# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
-
-# dependencies
-ui/node_modules
-ui/.pnp
-ui/.pnp.js
-
-# testing
-ui/coverage
-.cache_temp
-# production
-ui/build
-ui/.next
-
-# misc
-ui/.DS_Store
-ui/.env.local
-ui/.env.development.local
-ui/.env.test.local
-ui/.env.production.local
-
-ui/npm-debug.log*
-ui/yarn-debug.log*
-ui/yarn-error.log*
-ui/yarn.lock
-ui/package-lock.json
-
-#p2p download
-download/*
-
-#craft keys
-craft/key_files/*
-
-#bazel build output
-bazel-*
-/**/bazel-*

Cargo.toml

@@ -17,38 +17,39 @@ members = [
     "ceres",
     "libra",
 ]
+
 exclude = ["craft"]
 
 [dependencies]
 gateway = { path = "gateway" }
 common = { path = "common" }
 ceres = { path = "ceres" }
-config = "0.14"
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
 anyhow = { workspace = true }
 dotenvy = { workspace = true }
 tracing = { workspace = true }
 tracing-subscriber = { workspace = true }
-tracing-appender = { workspace = true}
+tracing-appender = { workspace = true }
 thiserror = { workspace = true }
 rand = { workspace = true }
 smallvec = { workspace = true }
 tokio = { workspace = true, features = ["macros"] }
 clap = { workspace = true, features = ["derive"] }
+config = "0.14"
 
 [dev-dependencies]
 reqwest = { version = "0.12.0", features = ["stream", "json"] }
-env_logger = "0.11.0"
-futures-util = "0.3.30"
-tokio-util = "0.7.10"
-git2 = "0.18.1"
-toml = "0.8.8"
 russh = { workspace = true }
 russh-keys = { workspace = true }
 async-trait = { workspace = true }
 bytes = { workspace = true }
 go-defer = { workspace = true }
+env_logger = "0.11.0"
+futures-util = "0.3.30"
+tokio-util = "0.7.10"
+git2 = "0.18.1"
+toml = "0.8.8"
 
 
 [workspace.dependencies]

README.md

@@ -6,7 +6,7 @@ Mega is an unofficial open source implementation of Google Piper. It is a monore
 
 Google Piper is a massive, centralized version control system that Google uses internally to manage their vast codebase. It is a monorepo, and a monolithic which mean is a single repository that contains all the source code for Google's software. It is designed to manage large-scale codebases, streamline development, and foster collaboration. It is built on top of Google's internal infrastructure and is designed to be highly scalable and efficient. More information on the [Why Google Stores Billions of Lines of Code in a Single Repository](https://cacm.acm.org/magazines/2016/7/204032-why-google-stores-billions-of-lines-of-code-in-a-single-repository/fulltext).
 
-:heavy_exclamation_mark: **Google Piper is not open source**
+**Google Piper is not open source**
 
 ## Mega features
 
@@ -22,11 +22,11 @@ When it comes to managing large codebases in a centralized manner, trunk-based d
 
 ### Conventional Commits
 
-Mega will  supports conventional commits, which are a set of rules for creating clear and concise commit messages.  More information on the [Conventional Commits](https://www.conventionalcommits.org/).
+Mega will support conventional commits, which are a set of rules for creating clear and concise commit messages.  More information on the [Conventional Commits](https://www.conventionalcommits.org/).
 
 ### Decentralized Open Source Collaboration
 
-For now, the entire open source community base on Git and GitHub. It's centralized model and it's not suitable for growing speed of open source world. Mega is working on build a decentralized open source collaboration model with [ZTM](https://github.com/flomesh-io/ztm)(Zero Trust Model) and decentralized social network like [Nostr](https://nostr.com), [Matrix](https://matrix.org) and [Mastodon](https://joinmastodon.org).
+For now, the entire open source community base on Git and GitHub. It's centralized model, and it's not suitable for growing speed of open source world. Mega is working on build a decentralized open source collaboration model with [ZTM](https://github.com/flomesh-io/ztm)(Zero Trust Model) and decentralized social network like [Nostr](https://nostr.com), [Matrix](https://matrix.org) and [Mastodon](https://joinmastodon.org).
 
 ## Quick Start
 
@@ -36,9 +36,20 @@ For now, we are developing on the macOS and Arch Linux. And quick start manuel i
 
 ### Quick Review of Architecture
 
+![Mega Architect](docs/images/architect.svg)
+
+1. **Gateway** - The Gateway module is responsible for handling `git`, `git-lfs` and web UI requests through the HTTP and SSH protocol. More information on the [Gateway](gateway/README.md).
+2. **Libra** - The Libra is a `git` program that rewrite in Rust. More information on the [Libra](libra/README.md).
+3. **Craft** - The Craft is `git` filters include `git-lfs` and `encrypt`/`decrypt` filer. More information on the [Craft](craft/README.md).
+4. **Gemini** - The Gemini is a decentralized module of Mega. More information on the [Gemini](gemini/README.md).
+5. **Scorpio** - The Scorpio is a FUSE filesystem that allow you to mount a Mega repository as a local filesystem. More information on the [Scorpio](scorpio/README.md).
+6. **Mercury** - The Mercury module is the core module of Mega, which rewrites Git internal object like Blob, Tree, etc. More information on the [Mercury Module](mercury/README.md).
+7. **Ceres** - The Ceres implement translate protocol for Mega. More information on the [Ceres](ceres/README.md).
+8. **Jupiter** - The Jupiter is storage engine for Mega. More information on the [Jupiter](jupiter/README.md).
+9. **Venus** - The Venus is a monorepo engine for Mega. More information on the [Venus](venus/README.md).
+10. **Mars** - The Mars is website for Mega. More information on the [Mars](mars/README.md).
+11. **Moon** - The Moon is a web UI for Mega. More information on the [Moon](moon/README.md).
 
-1. **Gateway Module** - The Gateway module is responsible for handling `git`, `git-lfs` and web UI requests througth the HTTP and SSH protocol. More information on the [Gateway Module](gateway/README.md).
-2. **Mercury Module** - The Mercury module is core module of Mega which rewrite Git internal object. More information on the [Mercury Module](mercury/README.md).
 
 ## Contributing
 
@@ -48,7 +59,7 @@ More information on contributing to Mega is available in the [Contributing Guide
 
 ## Talk and Share
 
-If you interested in Mega, you can make an appointment with us on [Google Calenader](https://calendar.app.google/QuBf2sdmf68wVYWL7) to disscuss your ideas, questions or problems and we will share our vision and roadmap with you.
+If you interested in Mega, you can make an appointment with us on [Google Calendar](https://calendar.app.google/QuBf2sdmf68wVYWL7) to discuss your ideas, questions or problems, and we will share our vision and roadmap with you.
 
 ## License
 

archived/README.md

@@ -9,11 +9,14 @@ This projects in archive folder has been officially archived. Please note the fo
 
 ## Background & Alternatives
 
-1. *ui* - The ui project will be replaced by a new project named [moon](moon/REAdME.md).
-2. *sync* - The sync project will be replaced by a new project named [moon-sync](moon-sync/REAdME.md).
-3. *build-bazel-tool* - The build-bazel-tool project will be replaced by a new project named [building](building/REAdME.md) in the feature.
-4. *website* - The website project will be replaced by a new project named [moon-website](moon-website/REAdME.md) in the feature.
-5. *mda* - The mda project still not have a replacement project.
+1. *ui* - The ui project will be replaced by a new project named [moon](../moon/README.md).
+2. *sync* - There are no replaced plan for the sync project.
+3. *build-bazel-tool* - The project will be replaced, but still not have a new project plan.
+4. *website* - The website project will be replaced by a new project named [Mars](../mars/README.md) in the feature.
+5. *mda* - The mda project still not have a replacement project. For now, we are working on decentralized LLM development.
+6. *fuse* - There are an alternate project plan to rewrite the **fuse** within [OSPP](https://summer-ospp.ac.cn) 2024 program, will be named [Scorpion](../scorpio/README.md).
+7. *kvcache* - The kvcache project still does not have a replacement project.
+8. *p2p* - The p2p project will be replaced by a new project named [Gemini](../gemini/README.md).
 
 [Optional: Suggest any alternative tools, libraries, or projects that users can consider as a replacement for this project.]
 

ceres/README.md

@@ -0,0 +1 @@
+## Ceres Module

common/README.md

@@ -0,0 +1 @@
+## Common Module

craft/README.md

@@ -1,10 +1,10 @@
-# Craft - Git Extension of Mega
+# Craft - Git Filters
 
 Craft is a Git plugin for Mega, a Large File Storage (LFS) client, encryption and decryption of code, and generation of AI/LLM training data or model data, among other functionalities. As an integral part of the Mega service, it is installed locally in the developers' environment and incorporated with the server to enhance developer experiences.
 
-## Quick Started for developing and testing craft on MacOS
+## Quick Started for developing and testing craft on macOS
 
-1. Install Rust on your MacOS.
+1. Install Rust on your macOS.
 
    ```bash
    $ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
@@ -56,14 +56,9 @@ Craft is a Git plugin for Mega, a Large File Storage (LFS) client, encryption an
 
 ### Usage
 
-1. `git-craft vault new-key [primary_id] [key_path]`
-  - git-craft will generate key with primary id and key name you entered to default file path
-2. `git-craft vault encrypt [key_path]`
-  - git-craft will get the file content and encrypt it, it should be used without public key path now, because I set a default key path
-3. `git-craft vault decrypt [key_path]`
-  - git-craft will decrypt blob data read from git's standard input stream, it should be used without secret key path now, because I set a default key path
-4. `git-craft vault list`
-  - git-craft will list keys name, key's fingerprint and id, it should be used without key path now, because I set a default key path
-5. `git-craft vault delete [key_path]`
-  - git-craft will show you what keys you have now, then remove keys by key name you entered, it should be used without key path now, because I set a default key path
+1. `git-craft vault new-key [primary_id] [key_path]` - git-craft will generate key with primary id and key name you entered to default file path
+2. `git-craft vault encrypt [key_path]` - git-craft will get the file content and encrypt it, it should be used without public key path now, because I set a default key path
+3. `git-craft vault decrypt [key_path]` - git-craft will decrypt blob data read from git's standard input stream, it should be used without secret key path now, because I set a default key path
+4. `git-craft vault list` - git-craft will list keys name, key's fingerprint and id, it should be used without key path now, because I set a default key path
+5. `git-craft vault delete [key_path]` - git-craft will show you what keys you have now, then remove keys by key name you entered, it should be used without key path now, because I set a default key path
 

delta/README.md

@@ -4,6 +4,7 @@ In Git, delta refers to the differences or changes between files or data objects
 ## Example
 
 This module exposes three functions to the outside world:
+
 ```rust
 pub fn delta_decode(mut stream : &mut impl Read,base_info: &Vec<u8>) -> Result<Vec<u8>, GitDeltaError>
 
@@ -20,9 +21,11 @@ let delta_result:Result<Vec<u8>, GitDeltaError> = delta::delta_decode(stream, ba
 ```
 
 On the contrary, if you want to compress another object in delta form based on it, use the encode function
+
 ```rust
 use delta;
 
 let delta_data:Vec<u8> = delta::delta_encode(old_data, new_data) ;
 ```
+
 In addition, the `delta::delta_encode_rate` function can represent the compression rate of delta

docs/api.md

@@ -18,7 +18,7 @@ HTTP implement for git transfer data between two repositories
     GET **/git-receive-pack
     ```
 
-3. When one Git repository wants to get data that a second repository has, the first can fetch from the second. This operation determines what data the server has that the client does not then streams that data down to the client in packfile format.
+3. When one Git repository wants to get data that a second repository has, the first can fetch from the second. This operation determines what data the server has that the client does not then streams that data down to the client in Pack file format.
 
     ```bash
     GET **/git-upload-pack
@@ -58,7 +58,7 @@ The Git LFS client uses an HTTPS server to coordinate fetching and storing large
     POST **/locks
     ```
 
-6. Delect Lock: The client can delete a lock, given its ID, by sending a POST to /locks/:id/unlock
+6. Delete Lock: The client can delete a lock, given its ID, by sending a POST to /locks/:id/unlock
 
     ```bash
     POST **/locks/:id/unlock

docs/development.md

@@ -2,7 +2,7 @@
 
 ## Architect
 
-![Mega Architect](images/mega-architect.png)
+![Mega Architect](images/architect.svg)
 
 ## Quick start manuel to developing or testing
 

ganymede/README.md

@@ -0,0 +1 @@
+## Ganymede Module

gateway/README.md

@@ -1,4 +1,4 @@
 ## Gateway Module
 
-The Gateway module is responsible for handling Git's `clone`, `push` and `pull` requests, the `git-lfs` client requests, and web UI requests. Most of requests are througth the HTTP protocol, and the Gateway module also support Git requests through SSH protocol.
+The Gateway module serves as the primary handler for various requests including Git's clone, push, and pull operations, git-lfs client interactions, and web UI requests. While the majority of these requests are facilitated via the HTTP protocol, the Gateway module is also equipped to process Git requests through the SSH protocol.
 

gemini/README.md

@@ -0,0 +1 @@
+## Gemini Module

jupiter/README.md

@@ -0,0 +1 @@
+## Jupiter Module

libra/README.md

@@ -0,0 +1 @@
+## Libra

mars/README.md

@@ -1,21 +1 @@
-# Mars - The Website for Mega
-
-
-## Getting started
-
-To get started with this template, first install the npm dependencies:
-
-```bash
-npm install
-```
-
-Next, run the development server:
-
-```bash
-npm run dev
-```
-
-Finally, open [http://localhost:3000](http://localhost:3000) in your browser to view the website.
-
-## License
-The source code is from Tailwind Template, and the template is a commercial product licensed under the [Tailwind UI license](LICENSE.md).
+## Mars Module

mercury/README.md

@@ -1,3 +1 @@
-## Mercury Module
-
-Mercury module is core module of Mega. 
+## Mercury Module

moon/README.md

@@ -1,36 +1 @@
-This is a [Next.js](https://nextjs.org/) project bootstrapped with [`create-next-app`](https://github.com/vercel/next.js/tree/canary/packages/create-next-app).
-
-## Getting Started
-
-First, run the development server:
-
-```bash
-npm run dev
-# or
-yarn dev
-# or
-pnpm dev
-# or
-bun dev
-```
-
-Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
-
-You can start editing the page by modifying `app/page.js`. The page auto-updates as you edit the file.
-
-This project uses [`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to automatically optimize and load Inter, a custom Google Font.
-
-## Learn More
-
-To learn more about Next.js, take a look at the following resources:
-
-- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
-- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
-
-You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js/) - your feedback and contributions are welcome!
-
-## Deploy on Vercel
-
-The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js.
-
-Check out our [Next.js deployment documentation](https://nextjs.org/docs/deployment) for more details.
+## Moon Module