Skip to content

feat: Support host-mode networking in automatic TF_CONFIG #287

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
inureyes merged 3 commits into from
Mar 16, 2023
Merged

feat: Support host-mode networking in automatic TF_CONFIG #287

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
908ccc1
feat: Support host-mode networking in automatic TF_CONFIG
achimnol Dec 7, 2022
2fb4967
fix: type matching and formatting
achimnol Dec 7, 2022
acd6d58
fix: update setup_multinode.py in other places
adrysn Dec 7, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 49 additions & 18 deletions python-ff/runner-scripts/setup_multinode.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,22 +1,53 @@
import os
import json
from pathlib import Path

if 'BACKENDAI_CLUSTER_HOST' in os.environ: # Start mutli-instance setup.
env = {}
env['cluster'] = {}
env['cluster']['worker'] = []
env['cluster']['chief'] = [] # For compatibility
env['cluster']['chief'].append('main1:2220')
for container in os.environ['BACKENDAI_CLUSTER_HOSTS'].split(","):
if container != 'main1':
env['cluster']['worker'].append(container + ":2220")
env['task'] = {}
if os.environ['BACKENDAI_CLUSTER_ROLE'] == 'main':
env['task']['type'] = "chief" # For compatibility. Recent TF choose first worker as chief.
env['task']["index"] = str(int(os.environ['BACKENDAI_CLUSTER_IDX']) - 1) # Index starts from 0

def get_ssh_port(mapping, cluster_host):
if mapping is None:
return 2220
try:
_, port = mapping[cluster_host] # host-ip and port
return port
except KeyError:
return 2220


if __name__ == "__main__":
if "BACKENDAI_CLUSTER_HOST" in os.environ: # Start mutli-instance setup.
try:
ssh_port_mapping = json.loads(Path("/home/config/ssh/port-mapping.json").read_bytes())
except FileNotFoundError:
ssh_port_mapping = None

env = {
"cluster": {
"chief": ["main1:" + str(get_ssh_port(ssh_port_mapping, "main1"))],
"worker": [],
},
}
for cluster_host in os.environ["BACKENDAI_CLUSTER_HOSTS"].split(","):
if cluster_host != "main1":
env["cluster"]["worker"].append(
cluster_host + ":" + str(get_ssh_port(ssh_port_mapping, cluster_host))
)

# TF's worker index starts from 0.
cluster_role_idx = str(int(os.environ["BACKENDAI_CLUSTER_IDX"]) - 1)
if os.environ["BACKENDAI_CLUSTER_ROLE"] == "main":
task = {
"type": "chief",
"index": cluster_role_idx,
}
else:
task = {
"type": "worker",
"index": cluster_role_idx,
}
env["task"] = task
print(json.dumps(env))
else:
env['task']['type'] = "worker"
env['task']["index"] = str(int(os.environ['BACKENDAI_CLUSTER_IDX']) - 1) # Index starts from 0
print(json.dumps(env))
else:
print("")
print("")


# vim: et sts=4 sw=4 tw=105
67 changes: 49 additions & 18 deletions python-pytorch/runner-scripts/setup_multinode.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,22 +1,53 @@
import os
import json
from pathlib import Path

if 'BACKENDAI_CLUSTER_HOST' in os.environ: # Start mutli-instance setup.
env = {}
env['cluster'] = {}
env['cluster']['worker'] = []
env['cluster']['chief'] = [] # For compatibility
env['cluster']['chief'].append('main1:2220')
for container in os.environ['BACKENDAI_CLUSTER_HOSTS'].split(","):
if container != 'main1':
env['cluster']['worker'].append(container + ":2220")
env['task'] = {}
if os.environ['BACKENDAI_CLUSTER_ROLE'] == 'main':
env['task']['type'] = "chief" # For compatibility. Recent TF choose first worker as chief.
env['task']["index"] = str(int(os.environ['BACKENDAI_CLUSTER_IDX']) - 1) # Index starts from 0

def get_ssh_port(mapping, cluster_host):
if mapping is None:
return 2220
try:
_, port = mapping[cluster_host] # host-ip and port
return port
except KeyError:
return 2220


if __name__ == "__main__":
if "BACKENDAI_CLUSTER_HOST" in os.environ: # Start mutli-instance setup.
try:
ssh_port_mapping = json.loads(Path("/home/config/ssh/port-mapping.json").read_bytes())
except FileNotFoundError:
ssh_port_mapping = None

env = {
"cluster": {
"chief": ["main1:" + str(get_ssh_port(ssh_port_mapping, "main1"))],
"worker": [],
},
}
for cluster_host in os.environ["BACKENDAI_CLUSTER_HOSTS"].split(","):
if cluster_host != "main1":
env["cluster"]["worker"].append(
cluster_host + ":" + str(get_ssh_port(ssh_port_mapping, cluster_host))
)

# TF's worker index starts from 0.
cluster_role_idx = str(int(os.environ["BACKENDAI_CLUSTER_IDX"]) - 1)
if os.environ["BACKENDAI_CLUSTER_ROLE"] == "main":
task = {
"type": "chief",
"index": cluster_role_idx,
}
else:
task = {
"type": "worker",
"index": cluster_role_idx,
}
env["task"] = task
print(json.dumps(env))
else:
env['task']['type'] = "worker"
env['task']["index"] = str(int(os.environ['BACKENDAI_CLUSTER_IDX']) - 1) # Index starts from 0
print(json.dumps(env))
else:
print("")
print("")


# vim: et sts=4 sw=4 tw=105
64 changes: 49 additions & 15 deletions python-pytorchlightning/runner-scripts/setup_multinode.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,19 +1,53 @@
import os
import json
from pathlib import Path

if 'BACKENDAI_CLUSTER_HOST' in os.environ: # Start mutli-instance setup.
env = {}
env['cluster'] = {}
env['cluster']['worker'] = []
for container in os.environ['BACKENDAI_CLUSTER_HOSTS'].split(","):
env['cluster']['worker'].append(container + ":2220")
env['task'] = {}
if os.environ['BACKENDAI_CLUSTER_ROLE'] == 'main':
env['task']['type'] = "worker" # Was chief. but recent TF choose first worker as chief.
env['task']["index"] = str(int(os.environ['BACKENDAI_CLUSTER_IDX']) - 1) # Index starts from 0

def get_ssh_port(mapping, cluster_host):
if mapping is None:
return 2220
try:
_, port = mapping[cluster_host] # host-ip and port
return port
except KeyError:
return 2220


if __name__ == "__main__":
if "BACKENDAI_CLUSTER_HOST" in os.environ: # Start mutli-instance setup.
try:
ssh_port_mapping = json.loads(Path("/home/config/ssh/port-mapping.json").read_bytes())
except FileNotFoundError:
ssh_port_mapping = None

env = {
"cluster": {
"chief": ["main1:" + str(get_ssh_port(ssh_port_mapping, "main1"))],
"worker": [],
},
}
for cluster_host in os.environ["BACKENDAI_CLUSTER_HOSTS"].split(","):
if cluster_host != "main1":
env["cluster"]["worker"].append(
cluster_host + ":" + str(get_ssh_port(ssh_port_mapping, cluster_host))
)

# TF's worker index starts from 0.
cluster_role_idx = str(int(os.environ["BACKENDAI_CLUSTER_IDX"]) - 1)
if os.environ["BACKENDAI_CLUSTER_ROLE"] == "main":
task = {
"type": "chief",
"index": cluster_role_idx,
}
else:
task = {
"type": "worker",
"index": cluster_role_idx,
}
env["task"] = task
print(json.dumps(env))
else:
env['task']['type'] = "worker"
env['task']["index"] = os.environ['BACKENDAI_CLUSTER_IDX']
print(json.dumps(env))
else:
print("")
print("")


# vim: et sts=4 sw=4 tw=105
67 changes: 49 additions & 18 deletions python-tensorflow/runner-scripts/setup_multinode.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,22 +1,53 @@
import os
import json
from pathlib import Path

if 'BACKENDAI_CLUSTER_HOST' in os.environ: # Start mutli-instance setup.
env = {}
env['cluster'] = {}
env['cluster']['worker'] = []
env['cluster']['chief'] = [] # For compatibility
env['cluster']['chief'].append('main1:2220')
for container in os.environ['BACKENDAI_CLUSTER_HOSTS'].split(","):
if container != 'main1':
env['cluster']['worker'].append(container + ":2220")
env['task'] = {}
if os.environ['BACKENDAI_CLUSTER_ROLE'] == 'main':
env['task']['type'] = "chief" # For compatibility. Recent TF choose first worker as chief.
env['task']["index"] = str(int(os.environ['BACKENDAI_CLUSTER_IDX']) - 1) # Index starts from 0

def get_ssh_port(mapping, cluster_host):
if mapping is None:
return 2220
try:
_, port = mapping[cluster_host] # host-ip and port
return port
except KeyError:
return 2220


if __name__ == "__main__":
if "BACKENDAI_CLUSTER_HOST" in os.environ: # Start mutli-instance setup.
try:
ssh_port_mapping = json.loads(Path("/home/config/ssh/port-mapping.json").read_bytes())
except FileNotFoundError:
ssh_port_mapping = None

env = {
"cluster": {
"chief": ["main1:" + str(get_ssh_port(ssh_port_mapping, "main1"))],
"worker": [],
},
}
for cluster_host in os.environ["BACKENDAI_CLUSTER_HOSTS"].split(","):
if cluster_host != "main1":
env["cluster"]["worker"].append(
cluster_host + ":" + str(get_ssh_port(ssh_port_mapping, cluster_host))
)

# TF's worker index starts from 0.
cluster_role_idx = str(int(os.environ["BACKENDAI_CLUSTER_IDX"]) - 1)
if os.environ["BACKENDAI_CLUSTER_ROLE"] == "main":
task = {
"type": "chief",
"index": cluster_role_idx,
}
else:
task = {
"type": "worker",
"index": cluster_role_idx,
}
env["task"] = task
print(json.dumps(env))
else:
env['task']['type'] = "worker"
env['task']["index"] = str(int(os.environ['BACKENDAI_CLUSTER_IDX']) - 1) # Index starts from 0
print(json.dumps(env))
else:
print("")
print("")


# vim: et sts=4 sw=4 tw=105
64 changes: 49 additions & 15 deletions vendor/ngc-pytorch/runner-scripts/setup_multinode.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,19 +1,53 @@
import os
import json
from pathlib import Path

if 'BACKENDAI_CLUSTER_HOST' in os.environ: # Start mutli-instance setup.
env = {}
env['cluster'] = {}
env['cluster']['worker'] = []
for container in os.environ['BACKENDAI_CLUSTER_HOSTS'].split(","):
env['cluster']['worker'].append(container + ":2220")
env['task'] = {}
if os.environ['BACKENDAI_CLUSTER_ROLE'] == 'main':
env['task']['type'] = "worker" # Was chief. but recent TF choose first worker as chief.
env['task']["index"] = str(int(os.environ['BACKENDAI_CLUSTER_IDX']) - 1) # Index starts from 0

def get_ssh_port(mapping, cluster_host):
if mapping is None:
return 2220
try:
_, port = mapping[cluster_host] # host-ip and port
return port
except KeyError:
return 2220


if __name__ == "__main__":
if "BACKENDAI_CLUSTER_HOST" in os.environ: # Start mutli-instance setup.
try:
ssh_port_mapping = json.loads(Path("/home/config/ssh/port-mapping.json").read_bytes())
except FileNotFoundError:
ssh_port_mapping = None

env = {
"cluster": {
"chief": ["main1:" + str(get_ssh_port(ssh_port_mapping, "main1"))],
"worker": [],
},
}
for cluster_host in os.environ["BACKENDAI_CLUSTER_HOSTS"].split(","):
if cluster_host != "main1":
env["cluster"]["worker"].append(
cluster_host + ":" + str(get_ssh_port(ssh_port_mapping, cluster_host))
)

# TF's worker index starts from 0.
cluster_role_idx = str(int(os.environ["BACKENDAI_CLUSTER_IDX"]) - 1)
if os.environ["BACKENDAI_CLUSTER_ROLE"] == "main":
task = {
"type": "chief",
"index": cluster_role_idx,
}
else:
task = {
"type": "worker",
"index": cluster_role_idx,
}
env["task"] = task
print(json.dumps(env))
else:
env['task']['type'] = "worker"
env['task']["index"] = os.environ['BACKENDAI_CLUSTER_IDX']
print(json.dumps(env))
else:
print("")
print("")


# vim: et sts=4 sw=4 tw=105
Loading