姑苏城外一茅屋，万树梅花月满天

0%

python | 进程池内的共享变量 Manager().dict()

发表于 更新于 分类于 ， ， ， ， Valine：

在此之前先看一下多进程的通信。

普通使用

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
import os
import time
from concurrent.futures.process import ProcessPoolExecutor
from multiprocessing import Manager


def job(v, k, lock):
    with lock:
        for i in range(10):
            value = v.setdefault(k, 0)
            time.sleep(0.1)
            v[k] = value + i
            print(f"{k} {os.getpid()} {value} {v}")


if __name__ == '__main__':
    v = Manager().dict()
    l = {}
    jobs = ["aa", "bb"]
    p = ProcessPoolExecutor(max_workers=5)
    for i in range(10):
        for k in jobs:
            lock = l.setdefault(k, Manager().Lock())
            p.submit(job, v, k, lock)

存储的是 int，可以运行一下，可以正常运行。

复杂对象的使用

如果 dict 里面存的是一个对象，那么会怎么样？

简单的复制

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
import time
from concurrent.futures.process import ProcessPoolExecutor
from dataclasses import dataclass
from multiprocessing import Manager


@dataclass
class Item:
    H: int = 1


def ChangeItem(v, item, i):
    try:
        a = v[i].H
        time.sleep(0.1)
        t = Item()
        t.H = a + i

        # v[i] = t
        
        # item = t

        v[i].H = v[i].H + a
        print(v)
    except Exception as e:
        print(e)


if __name__ == '__main__':
    v = Manager().dict()
    p = ProcessPoolExecutor(max_workers=5)
    for i in range(10):
        item = v.setdefault(i, Item())
        p.submit(ChangeItem, v, item, i)

    for i in range(10):
        item = v.setdefault(i, Item())
        p.submit(ChangeItem, v, item, i)

输出

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}

可以发现没有任何变化。

替换

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
import time
from concurrent.futures.process import ProcessPoolExecutor
from dataclasses import dataclass
from multiprocessing import Manager


@dataclass
class Item:
    H: int = 1


def ChangeItem(v, item, i):
    try:
        a = v[i].H
        time.sleep(0.1)
        t = Item()
        t.H = a + i

        # v[i] = t

        item = t

        # v[i].H = v[i].H + a
        print(v)
    except Exception as e:
        print(e)


if __name__ == '__main__':
    v = Manager().dict()
    p = ProcessPoolExecutor(max_workers=5)
    for i in range(10):
        item = v.setdefault(i, Item())
        p.submit(ChangeItem, v, item, i)

    for i in range(10):
        item = v.setdefault(i, Item())
        p.submit(ChangeItem, v, item, i)

输出

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}

可以发现也没啥变化。

重写

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
import time
from concurrent.futures.process import ProcessPoolExecutor
from dataclasses import dataclass
from multiprocessing import Manager


@dataclass
class Item:
    H: int = 1


def ChangeItem(v, item, i):
    try:
        a = v[i].H
        time.sleep(0.1)
        t = Item()
        t.H = a + i

        v[i] = t

        # item = t

        # v[i].H = v[i].H + a
        print(v)
    except Exception as e:
        print(e)


if __name__ == '__main__':
    v = Manager().dict()
    p = ProcessPoolExecutor(max_workers=5)
    for i in range(10):
        item = v.setdefault(i, Item())
        p.submit(ChangeItem, v, item, i)

    for i in range(10):
        item = v.setdefault(i, Item())
        p.submit(ChangeItem, v, item, i)

输出

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
{0: Item(H=1), 1: Item(H=1), 2: Item(H=1), 3: Item(H=1), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=2), 2: Item(H=3), 3: Item(H=4), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=2), 2: Item(H=3), 3: Item(H=4), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=2), 2: Item(H=3), 3: Item(H=4), 4: Item(H=1), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=2), 2: Item(H=3), 3: Item(H=4), 4: Item(H=5), 5: Item(H=1), 6: Item(H=1), 7: Item(H=1), 8: Item(H=1), 9: Item(H=1)}
{0: Item(H=1), 1: Item(H=2), 2: Item(H=3), 3: Item(H=4), 4: Item(H=5), 5: Item(H=6), 6: Item(H=7), 7: Item(H=8), 8: Item(H=9), 9: Item(H=10)}
{0: Item(H=1), 1: Item(H=2), 2: Item(H=3), 3: Item(H=4), 4: Item(H=5), 5: Item(H=6), 6: Item(H=7), 7: Item(H=8), 8: Item(H=9), 9: Item(H=10)}
{0: Item(H=1), 1: Item(H=2), 2: Item(H=3), 3: Item(H=4), 4: Item(H=5), 5: Item(H=6), 6: Item(H=7), 7: Item(H=8), 8: Item(H=9), 9: Item(H=10)}
{0: Item(H=1), 1: Item(H=2), 2: Item(H=3), 3: Item(H=4), 4: Item(H=5), 5: Item(H=6), 6: Item(H=7), 7: Item(H=8), 8: Item(H=9), 9: Item(H=10)}
{0: Item(H=1), 1: Item(H=2), 2: Item(H=3), 3: Item(H=4), 4: Item(H=5), 5: Item(H=6), 6: Item(H=7), 7: Item(H=8), 8: Item(H=9), 9: Item(H=10)}
{0: Item(H=1), 1: Item(H=2), 2: Item(H=3), 3: Item(H=4), 4: Item(H=5), 5: Item(H=6), 6: Item(H=7), 7: Item(H=8), 8: Item(H=9), 9: Item(H=10)}
{0: Item(H=1), 1: Item(H=3), 2: Item(H=3), 3: Item(H=4), 4: Item(H=5), 5: Item(H=6), 6: Item(H=7), 7: Item(H=8), 8: Item(H=9), 9: Item(H=10)}
{0: Item(H=1), 1: Item(H=3), 2: Item(H=5), 3: Item(H=7), 4: Item(H=9), 5: Item(H=6), 6: Item(H=7), 7: Item(H=8), 8: Item(H=9), 9: Item(H=10)}
{0: Item(H=1), 1: Item(H=3), 2: Item(H=5), 3: Item(H=7), 4: Item(H=9), 5: Item(H=6), 6: Item(H=7), 7: Item(H=8), 8: Item(H=9), 9: Item(H=10)}
{0: Item(H=1), 1: Item(H=3), 2: Item(H=5), 3: Item(H=7), 4: Item(H=9), 5: Item(H=6), 6: Item(H=7), 7: Item(H=8), 8: Item(H=9), 9: Item(H=10)}
{0: Item(H=1), 1: Item(H=3), 2: Item(H=5), 3: Item(H=7), 4: Item(H=9), 5: Item(H=11), 6: Item(H=7), 7: Item(H=8), 8: Item(H=9), 9: Item(H=10)}
{0: Item(H=1), 1: Item(H=3), 2: Item(H=5), 3: Item(H=7), 4: Item(H=9), 5: Item(H=11), 6: Item(H=13), 7: Item(H=8), 8: Item(H=9), 9: Item(H=10)}
{0: Item(H=1), 1: Item(H=3), 2: Item(H=5), 3: Item(H=7), 4: Item(H=9), 5: Item(H=11), 6: Item(H=13), 7: Item(H=15), 8: Item(H=17), 9: Item(H=10)}
{0: Item(H=1), 1: Item(H=3), 2: Item(H=5), 3: Item(H=7), 4: Item(H=9), 5: Item(H=11), 6: Item(H=13), 7: Item(H=15), 8: Item(H=17), 9: Item(H=19)}
{0: Item(H=1), 1: Item(H=3), 2: Item(H=5), 3: Item(H=7), 4: Item(H=9), 5: Item(H=11), 6: Item(H=13), 7: Item(H=15), 8: Item(H=17), 9: Item(H=19)}

可以发现现在确实是改变的。

请我喝杯咖啡吧～