Skip to content

Commit db83f49

Browse files
achoumcopybara-github
authored andcommitted
Internal change.
PiperOrigin-RevId: 1013686385
1 parent 6f4e048 commit db83f49

5 files changed

Lines changed: 164 additions & 118 deletions

File tree

‎dgf/src/bin/create_graph_sampling_plan.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,23 @@
7979
"If true and temporal sampling is enabled, the edgesets without a creation"
8080
" time get one derived from their connected nodes.",
8181
)
82+
_WITH_REPLACEMENT = flags.DEFINE_bool(
83+
"with_replacement",
84+
False,
85+
"If true, the sampled graphs are trees where the nodes visited several"
86+
" times are duplicated.",
87+
)
88+
_MULTI_VISIT = flags.DEFINE_bool(
89+
"multi_visit",
90+
True,
91+
"If true, a node reached several times at the same step of the plan is"
92+
" expanded once per visit.",
93+
)
94+
_MAX_TIMESERIES_LEN = flags.DEFINE_integer(
95+
"max_timeseries_len",
96+
32,
97+
"Maximum number of steps retained for each timeseries feature.",
98+
)
8299

83100

84101
def main(argv):
@@ -96,6 +113,9 @@ def main(argv):
96113
reverse=_REVERSE.value,
97114
temporal_sampling=_TEMPORAL_SAMPLING.value,
98115
propagate_timestamp_to_edges=_PROPAGATE_TIMESTAMP_TO_EDGES.value,
116+
with_replacement=_WITH_REPLACEMENT.value,
117+
multi_visit=_MULTI_VISIT.value,
118+
max_timeseries_len=_MAX_TIMESERIES_LEN.value,
99119
)
100120

101121
# Convert the config to a full sampling plan

‎dgf/src/sampling/offline_distributed/BUILD‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,16 +13,28 @@ package(
1313
# Libraries
1414
# =========
1515

16+
py_library(
17+
name = "common",
18+
srcs = ["common.py"],
19+
deps = [
20+
"//dgf/src/data:schema",
21+
"//dgf/src/io:schema",
22+
"//dgf/src/sampling:config",
23+
"//dgf/src/util:filesystem",
24+
"//dgf/src/util:log",
25+
],
26+
)
27+
1628
py_library(
1729
name = "offline_distributed_gcp",
1830
srcs = ["offline_distributed_gcp.py"],
1931
deps = [
32+
":common",
2033
# dataclasses_json dep,
2134
"//dgf/src/data:schema",
22-
"//dgf/src/io:schema",
2335
"//dgf/src/sampling:config",
24-
"//dgf/src/util:filesystem",
2536
"//dgf/src/util:log",
37+
"//dgf/src/util:util_py",
2638
"//dgf/src/util/weak_dep:weak_dep_aiplatform",
2739
# tqdm dep,
2840
],
@@ -41,6 +53,7 @@ py_test(
4153
name = "offline_distributed_gcp_test",
4254
srcs = ["offline_distributed_gcp_test.py"],
4355
deps = [
56+
":common",
4457
":offline_distributed_gcp",
4558
# absl/testing:absltest dep,
4659
"//dgf/src/sampling:config",
Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
# Copyright 2022 Google LLC.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""Utilities shared by the offline distributed samplers (GCP and Borg)."""
16+
17+
from dgf.src.data import schema as schema_lib
18+
from dgf.src.io import schema as io_schema
19+
from dgf.src.sampling import config as config_lib
20+
from dgf.src.util import filesystem
21+
from dgf.src.util import log
22+
23+
# Containers the input seeds can be read from. SSTable is excluded: it is a DAX
24+
# sink that the sampler cannot read seeds from.
25+
SUPPORTED_INPUT_SEED_CONTAINERS = frozenset({"TFRECORD", "RECORDIO"})
26+
27+
SAMPLING_CONFIG_FILENAME = "sampling_config.json"
28+
29+
30+
def validate_seed_args(
31+
num_seeds: int | None,
32+
num_samples_per_seed: int,
33+
input_seed_container: str,
34+
filter_seed_node: str | None = None,
35+
input_seeds: str | None = None,
36+
) -> None:
37+
"""Validates the arguments controlling the selection of the seeds.
38+
39+
Args:
40+
num_seeds: Number of seeds to select among the available ones, or None.
41+
num_samples_per_seed: Number of samples to generate for each seed.
42+
input_seed_container: Container of the input seeds.
43+
filter_seed_node: Optional filter on the seed nodes.
44+
input_seeds: Optional path to input seeds.
45+
46+
Raises:
47+
ValueError: If the arguments are inconsistent.
48+
"""
49+
if num_seeds is not None and num_seeds < 0:
50+
raise ValueError(f"num_seeds cannot be negative, got {num_seeds}.")
51+
if num_samples_per_seed < 1:
52+
raise ValueError(
53+
"num_samples_per_seed cannot be less than one, got"
54+
f" {num_samples_per_seed}."
55+
)
56+
if num_seeds and num_samples_per_seed > 1:
57+
raise ValueError(
58+
f"num_seeds ({num_seeds}) and num_samples_per_seed"
59+
f" ({num_samples_per_seed}) are exclusive: use num_seeds to generate"
60+
" fewer samples than there are seed nodes, and num_samples_per_seed to"
61+
" generate more."
62+
)
63+
if input_seed_container not in SUPPORTED_INPUT_SEED_CONTAINERS:
64+
raise ValueError(
65+
f"Unsupported input_seed_container {input_seed_container!r}. Supported"
66+
f" containers are {sorted(SUPPORTED_INPUT_SEED_CONTAINERS)}."
67+
)
68+
if filter_seed_node is not None and input_seeds is not None:
69+
raise ValueError(
70+
"filter_seed_node and input_seeds are exclusive: filter_seed_node only"
71+
" applies to the seed nodes taken from the root nodeset of the sampling"
72+
" plan."
73+
)
74+
75+
76+
def write_sampling_plan(
77+
input_path: str,
78+
output_path: str,
79+
plan: config_lib.SimpleSamplingConfig | config_lib.SamplingPlan,
80+
schema: schema_lib.GraphSchema | None,
81+
) -> str:
82+
"""Converts the sampling config to a plan if needed and writes it."""
83+
sampling_config_path = f"{output_path}/{SAMPLING_CONFIG_FILENAME}"
84+
85+
if isinstance(plan, config_lib.SimpleSamplingConfig):
86+
if schema is None:
87+
schema_path = f"{input_path}/schema.json"
88+
log.info("Reading graph schema from %s", schema_path)
89+
schema = io_schema.read_schema(schema_path)
90+
plan = config_lib.simple_sampling_config_to_sampling_plan(plan, schema)
91+
# Early check. The Go sampler validates the plan again.
92+
config_lib.check_supported_options(
93+
plan,
94+
schema,
95+
"offline distributed sampler",
96+
support_non_with_replacement=True,
97+
support_multi_visit=True,
98+
support_temporal_sampling=True,
99+
)
100+
101+
if not filesystem.is_gcs_path(output_path):
102+
filesystem.makedirs(output_path)
103+
log.info("Writing sampling plan to %s", sampling_config_path)
104+
with filesystem.open_write(sampling_config_path) as f:
105+
f.write(plan.to_json(indent=2)) # pyrefly: ignore[missing-attribute]
106+
107+
return sampling_config_path

‎dgf/src/sampling/offline_distributed/offline_distributed_gcp.py‎

Lines changed: 6 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,10 @@
2626

2727
import dataclasses_json
2828
from dgf.src.data import schema as schema_lib
29-
from dgf.src.io import schema as io_schema
3029
from dgf.src.sampling import config as config_lib
31-
from dgf.src.util import filesystem
30+
from dgf.src.sampling.offline_distributed import common
3231
from dgf.src.util import log
32+
from dgf.src.util import util
3333
from dgf.src.util.weak_dep.weak_dep_aiplatform import aiplatform
3434
import tqdm
3535

@@ -40,10 +40,6 @@
4040
_WORKER_MACHINE_TYPE = "n1-highmem-4"
4141
_DEFAULT_POLL_INTERVAL_SEC = 5.0
4242

43-
# Containers the input seeds can be read from. SSTable is excluded: it is a DAX
44-
# sink that the sampler cannot read seeds from.
45-
_SUPPORTED_INPUT_SEED_CONTAINERS = frozenset({"TFRECORD", "RECORDIO"})
46-
4743
_CONSOLE_JOB_URL_TEMPLATE = "https://console.cloud.google.com/vertex-ai/locations/{region}/training/{job_id}?project={project}"
4844
_CONSOLE_JOBS_LIST_URL_TEMPLATE = "https://console.cloud.google.com/vertex-ai/training/custom-jobs?project={project}"
4945
_LOGGING_JOB_URL_TEMPLATE = "https://console.cloud.google.com/logs/viewer?project={project}&resource=ml_job%2Fjob_id%2F{job_id}"
@@ -69,14 +65,6 @@ def _get_terminal_failure_states() -> set[Any]:
6965
]
7066

7167

72-
def _format_duration(seconds: float) -> str:
73-
"""Formats a duration in seconds into human-readable format."""
74-
sec = int(seconds)
75-
if sec >= 60:
76-
return f"{sec // 60}m {sec % 60:02d}s"
77-
return f"{sec}s"
78-
79-
8068
def _get_default_gcp_project() -> str | None:
8169
"""Gets the active GCP project from environment or gcloud config."""
8270
for env_var in (
@@ -121,52 +109,6 @@ def _validate_paths(input_path: str, output_path: str) -> tuple[str, str]:
121109
)
122110

123111

124-
def _validate_seed_args(
125-
num_seeds: int | None,
126-
num_samples_per_seed: int,
127-
input_seed_container: str,
128-
filter_seed_node: str | None = None,
129-
input_seeds: str | None = None,
130-
) -> None:
131-
"""Validates the arguments controlling the selection of the seeds.
132-
133-
Args:
134-
num_seeds: Number of seeds to select among the available ones, or None.
135-
num_samples_per_seed: Number of samples to generate for each seed.
136-
input_seed_container: Container of the input seeds.
137-
filter_seed_node: Optional filter on the seed nodes.
138-
input_seeds: Optional path to input seeds.
139-
140-
Raises:
141-
ValueError: If the arguments are inconsistent.
142-
"""
143-
if num_seeds is not None and num_seeds < 0:
144-
raise ValueError(f"num_seeds cannot be negative, got {num_seeds}.")
145-
if num_samples_per_seed < 1:
146-
raise ValueError(
147-
"num_samples_per_seed cannot be less than one, got"
148-
f" {num_samples_per_seed}."
149-
)
150-
if num_seeds and num_samples_per_seed > 1:
151-
raise ValueError(
152-
f"num_seeds ({num_seeds}) and num_samples_per_seed"
153-
f" ({num_samples_per_seed}) are exclusive: use num_seeds to generate"
154-
" fewer samples than there are seed nodes, and num_samples_per_seed to"
155-
" generate more."
156-
)
157-
if input_seed_container not in _SUPPORTED_INPUT_SEED_CONTAINERS:
158-
raise ValueError(
159-
f"Unsupported input_seed_container {input_seed_container!r}. Supported"
160-
f" containers are {sorted(_SUPPORTED_INPUT_SEED_CONTAINERS)}."
161-
)
162-
if filter_seed_node is not None and input_seeds is not None:
163-
raise ValueError(
164-
"filter_seed_node and input_seeds are exclusive: filter_seed_node only"
165-
" applies to the seed nodes taken from the root nodeset of the sampling"
166-
" plan."
167-
)
168-
169-
170112
def _get_job_urls(
171113
project: str, region: str, resource_name: str
172114
) -> tuple[str, str]:
@@ -201,38 +143,6 @@ def _get_job_stage_and_name(state: Any) -> tuple[int, str]:
201143
return 1, state_name
202144

203145

204-
def _write_sampling_plan(
205-
input_path: str,
206-
output_path: str,
207-
plan: config_lib.SimpleSamplingConfig | config_lib.SamplingPlan,
208-
schema: schema_lib.GraphSchema | None,
209-
) -> str:
210-
"""Converts sampling config to plan if needed and writes it to GCS."""
211-
sampling_config_path = f"{output_path}/sampling_config.json"
212-
213-
if isinstance(plan, config_lib.SimpleSamplingConfig):
214-
if schema is None:
215-
schema_path = f"{input_path}/schema.json"
216-
log.info("Reading graph schema from %s", schema_path)
217-
schema = io_schema.read_schema(schema_path)
218-
plan = config_lib.simple_sampling_config_to_sampling_plan(plan, schema)
219-
# Early check. The Go sampler validates the plan again.
220-
config_lib.check_supported_options(
221-
plan,
222-
schema,
223-
"offline distributed sampler",
224-
support_non_with_replacement=True,
225-
support_multi_visit=True,
226-
support_temporal_sampling=True,
227-
)
228-
229-
log.info("Writing sampling plan to %s", sampling_config_path)
230-
with filesystem.open_write(sampling_config_path) as f:
231-
f.write(plan.to_json(indent=2)) # pyrefly: ignore[missing-attribute]
232-
233-
return sampling_config_path
234-
235-
236146
@dataclasses_json.dataclass_json
237147
@dataclasses.dataclass
238148
class OfflineDistributedSamplingConfig:
@@ -373,7 +283,7 @@ def _monitor_job(
373283
while True:
374284
state = job.state
375285
stage_idx, state_name = _get_job_stage_and_name(state)
376-
elapsed_str = _format_duration(time.time() - start_time)
286+
elapsed_str = util.format_duration(time.time() - start_time)
377287

378288
if state_name != last_state_name:
379289
log.info(
@@ -537,7 +447,7 @@ def offline_distributed_sampler_gcp(
537447
be resolved.
538448
"""
539449
input_path, output_path = _validate_paths(input_path, output_path)
540-
_validate_seed_args(
450+
common.validate_seed_args(
541451
num_seeds=num_seeds,
542452
num_samples_per_seed=num_samples_per_seed,
543453
input_seed_container=input_seed_container,
@@ -566,7 +476,7 @@ def offline_distributed_sampler_gcp(
566476
display_name = f"dgf-distributed-sampler-glassbox-{timestamp}"
567477

568478
start_time = time.time()
569-
sampling_config_path = _write_sampling_plan(
479+
sampling_config_path = common.write_sampling_plan(
570480
input_path=input_path,
571481
output_path=output_path,
572482
plan=plan,
@@ -617,7 +527,7 @@ def offline_distributed_sampler_gcp(
617527
region=region,
618528
poll_interval=poll_interval,
619529
)
620-
total_duration = _format_duration(time.time() - start_time)
530+
total_duration = util.format_duration(time.time() - start_time)
621531
log.info("Total duration: %s", total_duration)
622532

623533
return job

0 commit comments

Comments
 (0)