[](https://discord.gg/qUtxnK2NMf)
# BitMoE
1 bit Mixture of Experts utilizing BitNet ++ Mixture of Experts. Also will add distribution amongst GPUs.
## install
`$ pip3 install bitmoe`
## usage
```python
import torch
from bitmoe.main import BitMoE
# Set the parameters
dim = 10 # Dimension of the input
hidden_dim = 20 # Dimension of the hidden layer
output_dim = 30 # Dimension of the output
num_experts = 5 # Number of experts in the BitMoE model
# Create the model
model = BitMoE(dim, hidden_dim, output_dim, num_experts)
# Create random inputs
batch_size = 32 # Number of samples in a batch
sequence_length = 100 # Length of the input sequence
x = torch.randn(batch_size, sequence_length, dim) # Random input tensor
# Forward pass
output = model(x) # Perform forward pass using the model
# Print the output shape
print(output) # Print the output tensor
print(output.shape) # Print the shape of the output tensor
```
# License
MIT
# Todo
- [ ] Implement better gating mechanisms
- [ ] Implement better routing algorithm
- [ ] Implement better BitFeedForward
- [ ] Implement
Raw data
{
"_id": null,
"home_page": "https://github.com/kyegomez/BitMoE",
"name": "bitmoe",
"maintainer": "",
"docs_url": null,
"requires_python": ">=3.6,<4.0",
"maintainer_email": "",
"keywords": "artificial intelligence,deep learning,optimizers,Prompt Engineering",
"author": "Kye Gomez",
"author_email": "kye@apac.ai",
"download_url": "https://files.pythonhosted.org/packages/39/23/870e948a1d07b9114c05dc2c488f83b2cd355717ec217a4c4c1d087dc8e1/bitmoe-0.0.2.tar.gz",
"platform": null,
"description": "[](https://discord.gg/qUtxnK2NMf)\n\n# BitMoE\n1 bit Mixture of Experts utilizing BitNet ++ Mixture of Experts. Also will add distribution amongst GPUs.\n\n## install\n`$ pip3 install bitmoe`\n\n## usage\n```python\nimport torch\nfrom bitmoe.main import BitMoE\n\n# Set the parameters\ndim = 10 # Dimension of the input\nhidden_dim = 20 # Dimension of the hidden layer\noutput_dim = 30 # Dimension of the output\nnum_experts = 5 # Number of experts in the BitMoE model\n\n# Create the model\nmodel = BitMoE(dim, hidden_dim, output_dim, num_experts)\n\n# Create random inputs\nbatch_size = 32 # Number of samples in a batch\nsequence_length = 100 # Length of the input sequence\nx = torch.randn(batch_size, sequence_length, dim) # Random input tensor\n\n# Forward pass\noutput = model(x) # Perform forward pass using the model\n\n# Print the output shape\nprint(output) # Print the output tensor\nprint(output.shape) # Print the shape of the output tensor\n```\n\n\n# License\nMIT\n\n\n# Todo\n\n- [ ] Implement better gating mechanisms\n- [ ] Implement better routing algorithm\n- [ ] Implement better BitFeedForward\n- [ ] Implement ",
"bugtrack_url": null,
"license": "MIT",
"summary": "BitMoE - Pytorch",
"version": "0.0.2",
"project_urls": {
"Documentation": "https://github.com/kyegomez/BitMoE",
"Homepage": "https://github.com/kyegomez/BitMoE",
"Repository": "https://github.com/kyegomez/BitMoE"
},
"split_keywords": [
"artificial intelligence",
"deep learning",
"optimizers",
"prompt engineering"
],
"urls": [
{
"comment_text": "",
"digests": {
"blake2b_256": "ba3e7521eede30ef5288a1a9ba9e07e41d4b2c8d59a8b501f88a83b942915bc6",
"md5": "bc783fdde1c4e5ffd70ef7a83d1ac462",
"sha256": "9663a03ce0376d392a6fc58d7ae2c73858914e6c9c47dc3a73696005ccacada3"
},
"downloads": -1,
"filename": "bitmoe-0.0.2-py3-none-any.whl",
"has_sig": false,
"md5_digest": "bc783fdde1c4e5ffd70ef7a83d1ac462",
"packagetype": "bdist_wheel",
"python_version": "py3",
"requires_python": ">=3.6,<4.0",
"size": 4141,
"upload_time": "2024-03-10T08:10:48",
"upload_time_iso_8601": "2024-03-10T08:10:48.344095Z",
"url": "https://files.pythonhosted.org/packages/ba/3e/7521eede30ef5288a1a9ba9e07e41d4b2c8d59a8b501f88a83b942915bc6/bitmoe-0.0.2-py3-none-any.whl",
"yanked": false,
"yanked_reason": null
},
{
"comment_text": "",
"digests": {
"blake2b_256": "3923870e948a1d07b9114c05dc2c488f83b2cd355717ec217a4c4c1d087dc8e1",
"md5": "0f48fb69ec121623786797dc7a7a4203",
"sha256": "dac5c228a9681ced7eb0b17351521a3583fc89aac7275cef338fb1f37532de83"
},
"downloads": -1,
"filename": "bitmoe-0.0.2.tar.gz",
"has_sig": false,
"md5_digest": "0f48fb69ec121623786797dc7a7a4203",
"packagetype": "sdist",
"python_version": "source",
"requires_python": ">=3.6,<4.0",
"size": 4275,
"upload_time": "2024-03-10T08:10:49",
"upload_time_iso_8601": "2024-03-10T08:10:49.965567Z",
"url": "https://files.pythonhosted.org/packages/39/23/870e948a1d07b9114c05dc2c488f83b2cd355717ec217a4c4c1d087dc8e1/bitmoe-0.0.2.tar.gz",
"yanked": false,
"yanked_reason": null
}
],
"upload_time": "2024-03-10 08:10:49",
"github": true,
"gitlab": false,
"bitbucket": false,
"codeberg": false,
"github_user": "kyegomez",
"github_project": "BitMoE",
"travis_ci": false,
"coveralls": false,
"github_actions": true,
"requirements": [
{
"name": "torch",
"specs": []
},
{
"name": "zetascale",
"specs": []
},
{
"name": "swarms",
"specs": []
}
],
"lcname": "bitmoe"
}
JSON
