# multiprocessing without __main__
## pip install multiprocnomain
## Advantages:
- Allows easy parallelization of functions without the need for execution in the '__\_\_main\_\___' block or at the top level.
- Facilitates the parallel execution of functions with different input parameters, providing flexibility.
- Improves performance by leveraging multiprocessing, especially for computationally intensive tasks.
- Automatically aggregates results into a dictionary, simplifying result mapping to input indices.
- Customizable parameters (processes and chunks) to adapt to specific requirements and hardware capabilities.
- Suitable for various use cases, including data science, machine learning, image processing, and scientific computing.
```python
Parameters:
- fu (callable): The function to be executed in parallel.
- it (iterable): An iterable of dictionaries, each containing the input parameters for the function.
- processes (int): The number of processes to use (default is 3).
- chunks (int): The chunk size for multiprocessing.Pool.starmap (default is 1).
Returns:
dict: A dictionary containing the results of the parallel executions, where keys correspond to the indices
of the input iterable and values contain the corresponding function outputs.
Examples:
import random
from multiprocnomain import start_multiprocessing
import subprocess
from a_cv_imwrite_imread_plus import open_image_in_cv
import numpy as np
def somefu(q=100):
exec(f"import random", globals()) # necessary
y = random.randint(10, 20)
for x in range(q):
y = y + x
return y
# somefu=lambda r:1111
it = [{"q": 100}, {"q": 100}, {"q": 100}, {"q": 10}]
b2 = start_multiprocessing(fu=somefu, it=it, processes=3, chunks=1)
print(b2)
def somefu2(path):
exec(f"import subprocess", globals()) # necessary
y = subprocess.run([f"ls" ,f"{path}"],capture_output=True)
return y
allpath=[{'path':'c:\\windows'}, {'path':'c:\\cygwin'}]
b1 = start_multiprocessing(fu=somefu2, it=allpath, processes=3, chunks=1)
print(b1)
def somefu3(q):
exec(f"from a_cv_imwrite_imread_plus import open_image_in_cv", globals()) # necessary
exec(f"import numpy as np", globals()) # necessary
im = open_image_in_cv(q)
r = im[..., 2]
g = im[..., 1]
b = im[..., 0]
return np.where((r == 255) & (g == 255) & (b == 255))
allimages = [
{"q": r"C:\Users\hansc\Pictures\collage_2023_04_23_06_04_51_956747.png"},
{"q": r"C:\Users\hansc\Pictures\bw_clickabutton.png"},
{"q": r"C:\Users\hansc\Pictures\cgea.png"},
{"q": r"C:\Users\hansc\Pictures\checkboxes.png"},
{"q": r"C:\Users\hansc\Pictures\clickabutton.png"},
{"q": r"C:\Users\hansc\Pictures\collage_2023_04_23_05_24_31_797203.png"},
{"q": r"C:\Users\hansc\Pictures\collage_2023_04_23_05_25_48_657510.png"},
{"q": r"C:\Users\hansc\Pictures\collage_2023_04_23_05_26_16_431863.png"},
{"q": r"C:\Users\hansc\Pictures\collage_2023_04_23_05_27_07_483808.png"},
{"q": r"C:\Users\hansc\Pictures\collage_2023_04_23_05_27_41_985343.png"},
{"q": r"C:\Users\hansc\Pictures\collage_2023_04_23_05_28_16_529438.png"},
{"q": r"C:\Users\hansc\Pictures\collage_2023_04_23_05_28_55_105250.png"},
{"q": r"C:\Users\hansc\Pictures\collage_2023_04_23_05_29_11_492492.png"},
{"q": r"C:\Users\hansc\Pictures\collage_2023_04_23_05_38_13_226848.png"},
{"q": r"C:\Users\hansc\Pictures\collage_2023_04_23_06_04_14_676085.png"},
{'q':r"C:\Users\hansc\Downloads\IMG-20230618-WA0000.jpeg"},
{'q':r"C:\Users\hansc\Downloads\maxresdefault.jpg"},
{'q':r"C:\Users\hansc\Downloads\panda-with-broom-600x500 (1).jpg"},
{'q':r"C:\Users\hansc\Downloads\panda-with-broom-600x500.jpg"},
{'q':r"C:\Users\hansc\Downloads\panda-with-broom-600x500222222222.jpg"},
{'q':r"C:\Users\hansc\Downloads\pexels-alex-andrews-2295744.jpg"},
{'q':r"C:\Users\hansc\Downloads\pexels-niki-nagy-1128416.jpg"},
]
b = start_multiprocessing(fu=somefu3, it=allimages, processes=3, chunks=5)
print(b)
```
Raw data
{
"_id": null,
"home_page": "https://github.com/hansalemaos/multiprocnomain",
"name": "multiprocnomain",
"maintainer": "",
"docs_url": null,
"requires_python": "",
"maintainer_email": "",
"keywords": "multiprocessing,cpus",
"author": "Johannes Fischer",
"author_email": "aulasparticularesdealemaosp@gmail.com",
"download_url": "https://files.pythonhosted.org/packages/63/fc/abb28cd597b8d575561ebf6fa44652f4bd125540e6f35b48ed1d6bbce053/multiprocnomain-0.11.tar.gz",
"platform": null,
"description": "\r\n# multiprocessing without __main__ \r\n\r\n## pip install multiprocnomain\r\n\r\n\r\n## Advantages:\r\n\r\n- Allows easy parallelization of functions without the need for execution in the '__\\_\\_main\\_\\___' block or at the top level.\r\n- Facilitates the parallel execution of functions with different input parameters, providing flexibility.\r\n- Improves performance by leveraging multiprocessing, especially for computationally intensive tasks.\r\n- Automatically aggregates results into a dictionary, simplifying result mapping to input indices.\r\n- Customizable parameters (processes and chunks) to adapt to specific requirements and hardware capabilities.\r\n- Suitable for various use cases, including data science, machine learning, image processing, and scientific computing.\r\n\r\n```python\r\n\r\nParameters:\r\n\t- fu (callable): The function to be executed in parallel.\r\n\t- it (iterable): An iterable of dictionaries, each containing the input parameters for the function.\r\n\t- processes (int): The number of processes to use (default is 3).\r\n\t- chunks (int): The chunk size for multiprocessing.Pool.starmap (default is 1).\r\n\r\nReturns:\r\n\tdict: A dictionary containing the results of the parallel executions, where keys correspond to the indices\r\n\t\tof the input iterable and values contain the corresponding function outputs.\r\n\r\nExamples:\r\n\timport random\r\n\tfrom multiprocnomain import start_multiprocessing\r\n\timport subprocess\r\n\tfrom a_cv_imwrite_imread_plus import open_image_in_cv\r\n\timport numpy as np\r\n\r\n\r\n\tdef somefu(q=100):\r\n\t\texec(f\"import random\", globals()) # necessary\r\n\t\ty = random.randint(10, 20)\r\n\t\tfor x in range(q):\r\n\t\t\ty = y + x\r\n\r\n\t\treturn y\r\n\r\n\r\n\t# somefu=lambda r:1111\r\n\tit = [{\"q\": 100}, {\"q\": 100}, {\"q\": 100}, {\"q\": 10}]\r\n\tb2 = start_multiprocessing(fu=somefu, it=it, processes=3, chunks=1)\r\n\tprint(b2)\r\n\r\n\r\n\tdef somefu2(path):\r\n\t\texec(f\"import subprocess\", globals()) # necessary\r\n\t\ty = subprocess.run([f\"ls\" ,f\"{path}\"],capture_output=True)\r\n\t\treturn y\r\n\r\n\tallpath=[{'path':'c:\\\\windows'}, {'path':'c:\\\\cygwin'}]\r\n\tb1 = start_multiprocessing(fu=somefu2, it=allpath, processes=3, chunks=1)\r\n\tprint(b1)\r\n\tdef somefu3(q):\r\n\t\texec(f\"from a_cv_imwrite_imread_plus import open_image_in_cv\", globals()) # necessary\r\n\t\texec(f\"import numpy as np\", globals()) # necessary\r\n\r\n\t\tim = open_image_in_cv(q)\r\n\t\tr = im[..., 2]\r\n\t\tg = im[..., 1]\r\n\t\tb = im[..., 0]\r\n\t\treturn np.where((r == 255) & (g == 255) & (b == 255))\r\n\r\n\r\n\tallimages = [\r\n\t\t{\"q\": r\"C:\\Users\\hansc\\Pictures\\collage_2023_04_23_06_04_51_956747.png\"},\r\n\t\t{\"q\": r\"C:\\Users\\hansc\\Pictures\\bw_clickabutton.png\"},\r\n\t\t{\"q\": r\"C:\\Users\\hansc\\Pictures\\cgea.png\"},\r\n\t\t{\"q\": r\"C:\\Users\\hansc\\Pictures\\checkboxes.png\"},\r\n\t\t{\"q\": r\"C:\\Users\\hansc\\Pictures\\clickabutton.png\"},\r\n\t\t{\"q\": r\"C:\\Users\\hansc\\Pictures\\collage_2023_04_23_05_24_31_797203.png\"},\r\n\t\t{\"q\": r\"C:\\Users\\hansc\\Pictures\\collage_2023_04_23_05_25_48_657510.png\"},\r\n\t\t{\"q\": r\"C:\\Users\\hansc\\Pictures\\collage_2023_04_23_05_26_16_431863.png\"},\r\n\t\t{\"q\": r\"C:\\Users\\hansc\\Pictures\\collage_2023_04_23_05_27_07_483808.png\"},\r\n\t\t{\"q\": r\"C:\\Users\\hansc\\Pictures\\collage_2023_04_23_05_27_41_985343.png\"},\r\n\t\t{\"q\": r\"C:\\Users\\hansc\\Pictures\\collage_2023_04_23_05_28_16_529438.png\"},\r\n\t\t{\"q\": r\"C:\\Users\\hansc\\Pictures\\collage_2023_04_23_05_28_55_105250.png\"},\r\n\t\t{\"q\": r\"C:\\Users\\hansc\\Pictures\\collage_2023_04_23_05_29_11_492492.png\"},\r\n\t\t{\"q\": r\"C:\\Users\\hansc\\Pictures\\collage_2023_04_23_05_38_13_226848.png\"},\r\n\t\t{\"q\": r\"C:\\Users\\hansc\\Pictures\\collage_2023_04_23_06_04_14_676085.png\"},\r\n\t\t{'q':r\"C:\\Users\\hansc\\Downloads\\IMG-20230618-WA0000.jpeg\"},\r\n\t\t{'q':r\"C:\\Users\\hansc\\Downloads\\maxresdefault.jpg\"},\r\n\t\t{'q':r\"C:\\Users\\hansc\\Downloads\\panda-with-broom-600x500 (1).jpg\"},\r\n\t\t{'q':r\"C:\\Users\\hansc\\Downloads\\panda-with-broom-600x500.jpg\"},\r\n\t\t{'q':r\"C:\\Users\\hansc\\Downloads\\panda-with-broom-600x500222222222.jpg\"},\r\n\t\t{'q':r\"C:\\Users\\hansc\\Downloads\\pexels-alex-andrews-2295744.jpg\"},\r\n\t\t{'q':r\"C:\\Users\\hansc\\Downloads\\pexels-niki-nagy-1128416.jpg\"},\r\n\r\n\t]\r\n\tb = start_multiprocessing(fu=somefu3, it=allimages, processes=3, chunks=5)\r\n\tprint(b)\r\n\r\n\r\n```\r\n",
"bugtrack_url": null,
"license": "MIT",
"summary": "multiprocessing without __main__",
"version": "0.11",
"project_urls": {
"Homepage": "https://github.com/hansalemaos/multiprocnomain"
},
"split_keywords": [
"multiprocessing",
"cpus"
],
"urls": [
{
"comment_text": "",
"digests": {
"blake2b_256": "5becd9e80be5072e25636f9ade34c07f516c7733cbb788d948fd2a0cdb731a94",
"md5": "7527faf635cb03ec57e3724b38194edf",
"sha256": "cf7dd879aefa2721365d4d1dc2ee960104196646839ac3757cd8064ba956bbe8"
},
"downloads": -1,
"filename": "multiprocnomain-0.11-py3-none-any.whl",
"has_sig": false,
"md5_digest": "7527faf635cb03ec57e3724b38194edf",
"packagetype": "bdist_wheel",
"python_version": "py3",
"requires_python": null,
"size": 8859,
"upload_time": "2023-11-11T03:05:57",
"upload_time_iso_8601": "2023-11-11T03:05:57.084216Z",
"url": "https://files.pythonhosted.org/packages/5b/ec/d9e80be5072e25636f9ade34c07f516c7733cbb788d948fd2a0cdb731a94/multiprocnomain-0.11-py3-none-any.whl",
"yanked": false,
"yanked_reason": null
},
{
"comment_text": "",
"digests": {
"blake2b_256": "63fcabb28cd597b8d575561ebf6fa44652f4bd125540e6f35b48ed1d6bbce053",
"md5": "8aad9246e97e7aa365e8ed9e9538b6a8",
"sha256": "29c7e66c93ffdadc2808fa873ab1c53ebff34c2fbeb47becb6c0c4610d762333"
},
"downloads": -1,
"filename": "multiprocnomain-0.11.tar.gz",
"has_sig": false,
"md5_digest": "8aad9246e97e7aa365e8ed9e9538b6a8",
"packagetype": "sdist",
"python_version": "source",
"requires_python": null,
"size": 6810,
"upload_time": "2023-11-11T03:05:58",
"upload_time_iso_8601": "2023-11-11T03:05:58.750255Z",
"url": "https://files.pythonhosted.org/packages/63/fc/abb28cd597b8d575561ebf6fa44652f4bd125540e6f35b48ed1d6bbce053/multiprocnomain-0.11.tar.gz",
"yanked": false,
"yanked_reason": null
}
],
"upload_time": "2023-11-11 03:05:58",
"github": true,
"gitlab": false,
"bitbucket": false,
"codeberg": false,
"github_user": "hansalemaos",
"github_project": "multiprocnomain",
"travis_ci": false,
"coveralls": false,
"github_actions": false,
"requirements": [],
"lcname": "multiprocnomain"
}
JSON
