Bazel support (#12)

* build rust

* go builds

* tests pass

* release runs too

* gazelle formatting

* add CI

* check in cargo.lock based on this https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html

* add linker opts

* does renaming help?

* builds

* upgrade to 0.14.1

* update release files

.github/workflows/ci.yaml

@@ -0,0 +1,32 @@
+name: CI
+
+on:
+  push: {}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+
+    - name: Mount bazel cache
+      uses: actions/cache@v1
+      with:
+        path: "/home/runner/.cache/bazel"
+        key: bazel
+
+    - name: Install bazelisk
+      run: |
+        curl -LO "https://github.com/bazelbuild/bazelisk/releases/download/v1.18.0/bazelisk-linux-amd64"
+        mkdir -p "${GITHUB_WORKSPACE}/bin/"
+        mv bazelisk-linux-amd64 "${GITHUB_WORKSPACE}/bin/bazel"
+        chmod +x "${GITHUB_WORKSPACE}/bin/bazel"
+
+    - name: Test
+      run: |
+        "${GITHUB_WORKSPACE}/bin/bazel" test //...
+
+    - name: Build
+      run: |
+        "${GITHUB_WORKSPACE}/bin/bazel" build //...

.gitignore

@@ -1,2 +1,8 @@
 libtokenizers.a
 /artifacts
+
+# Rust
+target
+
+# Bazel
+/bazel-*

BUILD.bazel

@@ -0,0 +1,58 @@
+load("@bazel_gazelle//:def.bzl", "gazelle")
+load("@crate_index//:defs.bzl", "aliases", "all_crate_deps")
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+load("@rules_rust//rust:defs.bzl", "rust_static_library")
+
+# gazelle:prefix github.com/daulet/tokenizers
+gazelle(
+    name = "gazelle",
+)
+
+gazelle(
+    name = "gazelle-update-repos",
+    args = [
+        "-from_file=go.mod",
+        "-to_macro=deps.bzl%go_dependencies",
+        "-prune",
+    ],
+    command = "update-repos",
+)
+
+rust_static_library(
+    name = "tokenizers",
+    srcs = glob([
+        "src/**/*.rs",
+    ]),
+    aliases = aliases(),
+    proc_macro_deps = all_crate_deps(
+        proc_macro = True,
+    ),
+    visibility = ["//visibility:public"],
+    deps = all_crate_deps(),
+)
+
+go_test(
+    name = "tokenizers_test",
+    srcs = ["tokenizer_test.go"],
+    data = ["//test:data"],
+    embedsrcs = ["//test:embeddata"],
+    deps = [
+        ":tokenizers_go",
+        "@com_github_stretchr_testify//assert",
+        "@com_github_stretchr_testify//require",
+    ],
+)
+
+go_library(
+    name = "tokenizers_go",
+    srcs = [
+        "tokenizer.go",
+        "tokenizers.h",
+    ],
+    cdeps = [
+        ":tokenizers",
+    ],
+    cgo = True,
+    importpath = "github.com/daulet/tokenizers",
+    visibility = ["//visibility:public"],
+)