From bd146aac4a874b47b3bbd383150d6c9675e72a73 Mon Sep 17 00:00:00 2001
From: Marie Weiel <marie.weiel@kit.edu>
Date: Mon, 2 Dec 2024 09:24:34 +0100
Subject: [PATCH] add solutions
---
2_psrs_cdist/solutions/A1/cdist.py | 150 +
2_psrs_cdist/solutions/A1/cdist_symmetric.py | 216 ++
.../solutions/A1/cdist_symmetric_weak.py | 226 ++
2_psrs_cdist/solutions/A1/cdist_weak.py | 159 +
2_psrs_cdist/solutions/A1/plot_scaling.py | 172 +
.../solutions/A1/strong_scaling/nodes_1.out | 136 +
.../solutions/A1/strong_scaling/nodes_16.out | 1335 +++++++
.../solutions/A1/strong_scaling/nodes_2.out | 257 ++
.../solutions/A1/strong_scaling/nodes_32.out | 2567 ++++++++++++++
.../solutions/A1/strong_scaling/nodes_4.out | 411 +++
.../solutions/A1/strong_scaling/nodes_8.out | 719 ++++
.../A1/strong_scaling/strong_scaling.pdf | Bin 0 -> 17643 bytes
.../A1/strong_scaling_symmetric/nodes_1.out | 136 +
.../A1/strong_scaling_symmetric/nodes_16.out | 2743 +++++++++++++++
.../A1/strong_scaling_symmetric/nodes_2.out | 279 ++
.../A1/strong_scaling_symmetric/nodes_4.out | 499 +++
.../A1/strong_scaling_symmetric/nodes_8.out | 1071 ++++++
.../strong_scaling_symmetric.pdf | Bin 0 -> 17608 bytes
.../A1/submit_strong_scaling_example.sh | 27 +
.../A1/submit_weak_scaling_example.py | 27 +
.../solutions/A1/weak_scaling/nodes_1.out | 158 +
.../solutions/A1/weak_scaling/nodes_16.out | 1687 +++++++++
.../solutions/A1/weak_scaling/nodes_2.out | 301 ++
.../solutions/A1/weak_scaling/nodes_4.out | 499 +++
.../solutions/A1/weak_scaling/nodes_8.out | 895 +++++
.../A1/weak_scaling/weak_scaling.pdf | Bin 0 -> 17695 bytes
.../A1/weak_scaling_symmetric/nodes_1.out | 158 +
.../A1/weak_scaling_symmetric/nodes_16.out | 3095 +++++++++++++++++
.../A1/weak_scaling_symmetric/nodes_2.out | 323 ++
.../A1/weak_scaling_symmetric/nodes_4.out | 587 ++++
.../A1/weak_scaling_symmetric/nodes_8.out | 1247 +++++++
.../weak_scaling_symmetric.pdf | Bin 0 -> 18377 bytes
2_psrs_cdist/solutions/A2/psrs.py | 266 ++
2_psrs_cdist/solutions/A2/psrs_job.out | 324 ++
34 files changed, 20670 insertions(+)
create mode 100644 2_psrs_cdist/solutions/A1/cdist.py
create mode 100644 2_psrs_cdist/solutions/A1/cdist_symmetric.py
create mode 100644 2_psrs_cdist/solutions/A1/cdist_symmetric_weak.py
create mode 100644 2_psrs_cdist/solutions/A1/cdist_weak.py
create mode 100644 2_psrs_cdist/solutions/A1/plot_scaling.py
create mode 100644 2_psrs_cdist/solutions/A1/strong_scaling/nodes_1.out
create mode 100644 2_psrs_cdist/solutions/A1/strong_scaling/nodes_16.out
create mode 100644 2_psrs_cdist/solutions/A1/strong_scaling/nodes_2.out
create mode 100644 2_psrs_cdist/solutions/A1/strong_scaling/nodes_32.out
create mode 100644 2_psrs_cdist/solutions/A1/strong_scaling/nodes_4.out
create mode 100644 2_psrs_cdist/solutions/A1/strong_scaling/nodes_8.out
create mode 100644 2_psrs_cdist/solutions/A1/strong_scaling/strong_scaling.pdf
create mode 100644 2_psrs_cdist/solutions/A1/strong_scaling_symmetric/nodes_1.out
create mode 100644 2_psrs_cdist/solutions/A1/strong_scaling_symmetric/nodes_16.out
create mode 100644 2_psrs_cdist/solutions/A1/strong_scaling_symmetric/nodes_2.out
create mode 100644 2_psrs_cdist/solutions/A1/strong_scaling_symmetric/nodes_4.out
create mode 100644 2_psrs_cdist/solutions/A1/strong_scaling_symmetric/nodes_8.out
create mode 100644 2_psrs_cdist/solutions/A1/strong_scaling_symmetric/strong_scaling_symmetric.pdf
create mode 100644 2_psrs_cdist/solutions/A1/submit_strong_scaling_example.sh
create mode 100644 2_psrs_cdist/solutions/A1/submit_weak_scaling_example.py
create mode 100644 2_psrs_cdist/solutions/A1/weak_scaling/nodes_1.out
create mode 100644 2_psrs_cdist/solutions/A1/weak_scaling/nodes_16.out
create mode 100644 2_psrs_cdist/solutions/A1/weak_scaling/nodes_2.out
create mode 100644 2_psrs_cdist/solutions/A1/weak_scaling/nodes_4.out
create mode 100644 2_psrs_cdist/solutions/A1/weak_scaling/nodes_8.out
create mode 100644 2_psrs_cdist/solutions/A1/weak_scaling/weak_scaling.pdf
create mode 100644 2_psrs_cdist/solutions/A1/weak_scaling_symmetric/nodes_1.out
create mode 100644 2_psrs_cdist/solutions/A1/weak_scaling_symmetric/nodes_16.out
create mode 100644 2_psrs_cdist/solutions/A1/weak_scaling_symmetric/nodes_2.out
create mode 100644 2_psrs_cdist/solutions/A1/weak_scaling_symmetric/nodes_4.out
create mode 100644 2_psrs_cdist/solutions/A1/weak_scaling_symmetric/nodes_8.out
create mode 100644 2_psrs_cdist/solutions/A1/weak_scaling_symmetric/weak_scaling_symmetric.pdf
create mode 100644 2_psrs_cdist/solutions/A2/psrs.py
create mode 100644 2_psrs_cdist/solutions/A2/psrs_job.out
diff --git a/2_psrs_cdist/solutions/A1/cdist.py b/2_psrs_cdist/solutions/A1/cdist.py
new file mode 100644
index 0000000..b5a67a5
--- /dev/null
+++ b/2_psrs_cdist/solutions/A1/cdist.py
@@ -0,0 +1,150 @@
+"""Parallel calculation of pairwise distances"""
+import time
+
+import h5py
+import torch
+from mpi4py import MPI
+
+torch.set_default_dtype(torch.float32)
+
+
+def dist(x: torch.Tensor, y: torch.Tensor, comm: MPI.Comm = MPI.COMM_WORLD) -> torch.Tensor:
+ """
+ Calculate pairwise distances between all rows (samples, i.e., along axis 0) of two tensors x and y in parallel.
+
+ The distance matrix is calculated tile-wise with ring communication between processes, each holding a piece of x
+ and/or y.
+
+ Parameters
+ ----------
+ x : torch.Tensor
+ First 2d tensor (of shape m/p x f). m is the total number of samples in x, distributed over p processors.
+ f is the number of features.
+ y : torch.Tensor
+ Second 2d tensor (of shape n/p x f). n is the total number of samples in x, distributed over p processors.
+ The number of features f must be the same as for x.
+ comm : MPI.Comm
+ Communicator to use. Default is ``MPI.COMM_WORLD``.
+ """
+ # Check whether two input tensors are compatible.
+ if len(x.shape) != len(y.shape) != 2:
+ raise ValueError("Input tensors must be two-dimensional.")
+ if x.shape[1] != y.shape[1]:
+ raise ValueError(f"Input tensors must have the same number of features but {x.shape[1]} != {y.shape[1]}.")
+
+ size, rank = comm.size, comm.rank # Set up communication.
+
+ if size == 1: # Use torch functionality in non-parallel case.
+ return torch.cdist(x, y)
+
+ else: # Parallel case
+ # --- Setup and Matrix Initialization ---
+ mp, f = x.shape # Get number of samples in local chunk of x and number of features.
+ np = y.shape[0] # Get number of samples in local chunk of y.
+
+ # Each process initializes a local matrix, `local_distances`, of shape `(mp, n)`, where `mp` is the local chunk
+ # size of `x`, and `n` is the total number of samples in `y`. Each rank thus calculates the distance matrix
+ # chunk of size `mp x n`, i.e., rank 0 has distances from its own local `x` to all other `y`'s.
+
+ # Determine overall number of samples in y.
+ n = comm.allreduce(np, op=MPI.SUM)
+ print(f"Overall number of samples is {n}.")
+ # Initialize rank-local chunk of mp x n distance matrix with zeros.
+ local_distances = torch.zeros((mp, n))
+
+ # --- Managing Chunks and Displacements ---
+ # Determine where to put each result in the rank-local distance matrix chunk.
+ # Determine number of samples (rows) in each rank-local y.
+ y_counts = torch.tensor(comm.allgather(np), dtype=torch.int)
+ # Calculate corresponding displacements from counts to record the starting index of each chunk in y. Thus, each
+ # process can identify where in the result matrix it should write the distances.
+ y_displ = (0,) + tuple(torch.cumsum(y_counts, dim=0, dtype=torch.int)[:-1])
+
+ # --- Ring Communication Pattern ---
+ # Calculate distances in a "ring" pattern. Each process calculates distances for its local x chunk against its
+ # local y chunk (diagonal calculation). Then, through `size - 1` iterations, each process sends its y chunk to
+ # the next process in the "ring" while receiving a new y chunk from the previous process. This continues until
+ # each process has calculated distances between its x chunk and all chunks of y across all processes.
+
+ # 0th iteration: Calculate diagonal of global distance matrix.
+ # Each process calculates distances for its local x chunk against its local y chunk.
+ print(f"Rank [{rank}/{size}]: Calculate diagonal blocks in global distance matrix...")
+ # Extract actual result columns in distance matrix chunk for each rank.
+ cols = (y_displ[rank], y_displ[rank + 1] if (rank + 1) != size else n)
+ d_ij = torch.cdist(x, y)
+ local_distances[:, cols[0]: cols[1]] = d_ij
+
+ print(f"Rank [{rank}/{size}]: Start tile-wise ring communication...")
+ # Remaining `size-1` iterations: Send rank-local part of y to next process in circular fashion.
+ for iter_idx in range(1, size):
+
+ receiver = (rank + iter_idx) % size # Determine receiving process.
+ sender = (rank - iter_idx) % size # Determine sending process.
+ # Determine columns of rank-local distance matrix chunk to write result to.
+ col1 = y_displ[sender]
+ col2 = y_displ[sender + 1] if sender != size - 1 else n
+ # All but first `iter_idx` processes are first receiving, then sending.
+ if (rank // iter_idx) != 0:
+ stat = MPI.Status()
+ # Probe for incoming message containing the next chunk of y to consider.
+ comm.Probe(source=sender, tag=iter_idx, status=stat)
+ # Determine number of samples to receive (= overall number of floats to receive / number of features).
+ count = int(stat.Get_count(MPI.FLOAT) / f)
+ # Initialize tensor for incoming chunk of y with zeros.
+ moving = torch.zeros((count, f))
+ comm.Recv(moving, source=sender, tag=iter_idx)
+ # Send rank-local chunk of y to next process.
+ comm.Send(y, dest=receiver, tag=iter_idx)
+ # First `iter_idx` processes can now receive after sending.
+ if (rank // iter_idx) == 0:
+ stat = MPI.Status()
+ comm.Probe(source=sender, tag=iter_idx, status=stat)
+ count = int(stat.Get_count(MPI.FLOAT) / f)
+ moving = torch.zeros((count, f))
+ comm.Recv(moving, source=sender, tag=iter_idx)
+ # Calculate distances between stationary chunk of x and currently considered, moving chunk of y.
+ # Write result at correct position in distance matrix.
+ d_ij = torch.cdist(x, moving)
+ local_distances[:, col1: col2] = d_ij
+ print(f"Rank [{rank}/{size}]: [DONE]")
+
+ return local_distances
+
+
+if __name__ == "__main__":
+ comm = MPI.COMM_WORLD
+ rank, size = comm.rank, comm.size
+
+ data_path = "/pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5"
+ dataset = "data"
+
+ if rank == 0:
+ print(
+ "######################\n"
+ "# Pairwise distances #\n"
+ "######################\n"
+ f"COMM_WORLD size is {size}.\n"
+ f"Loading data... {data_path}[{dataset}]"
+ )
+
+ # Parallel data loader for SUSY data.
+ with h5py.File(data_path, "r") as handle:
+ chunk = int(handle[dataset].shape[0]/size)
+ if rank == size - 1:
+ data = torch.FloatTensor(handle[dataset][rank*chunk:])
+ else:
+ data = torch.FloatTensor(handle[dataset][rank*chunk:(rank+1)*chunk])
+
+ print(f"\t[OK]\nRank [{rank}/{size}]: Local data chunk has shape {list(data.shape)}...")
+
+ if rank == 0:
+ print("Start distance calculations...")
+ # Calculate distances of all SUSY samples w.r.t. each other and measure runtime.
+ start = time.perf_counter()
+ distances = dist(data, data, comm)
+ local_runtime = time.perf_counter() - start
+ # Calculate process-averaged runtime.
+ average_runtime = comm.allreduce(local_runtime, op=MPI.SUM) / size
+ print(f"Rank [{rank}/{size}]: Local distance matrix has shape {list(distances.shape)}.")
+ if rank == 0:
+ print(f"Process-averaged run time:\t{average_runtime} s")
diff --git a/2_psrs_cdist/solutions/A1/cdist_symmetric.py b/2_psrs_cdist/solutions/A1/cdist_symmetric.py
new file mode 100644
index 0000000..1c60615
--- /dev/null
+++ b/2_psrs_cdist/solutions/A1/cdist_symmetric.py
@@ -0,0 +1,216 @@
+import time
+
+import h5py
+import torch
+from mpi4py import MPI
+
+torch.set_default_dtype(torch.float32)
+
+
+def dist_symmetric(x: torch.Tensor, y: torch.Tensor, comm: MPI.Comm = MPI.COMM_WORLD) -> torch.Tensor:
+ """
+ Calculate pairwise distances between all rows (samples, i.e., along axis 0) of two tensors x and y in parallel.
+
+ The distance matrix is calculated tile-wise with ring communication between processes, each holding a piece of x
+ and/or y.
+
+ Parameters
+ ----------
+ x : torch.Tensor
+ First 2d tensor (of shape m/p x f). m is the total number of samples in x, distributed over p processors.
+ f is the number of features.
+ y : torch.Tensor
+ Second 2d tensor (of shape n/p x f). n is the total number of samples in x, distributed over p processors.
+ The number of features f must be the same as for x.
+ comm : MPI.Comm
+ Communicator to use. Default is ``MPI.COMM_WORLD``.
+ """
+ # Check whether two input tensors are compatible.
+ if len(x.shape) != len(y.shape) != 2:
+ raise ValueError("Input tensors must be two-dimensional.")
+ if x.shape[1] != y.shape[1]:
+ raise ValueError(f"Input tensors must have the same number of features but {x.shape[1]} != {y.shape[1]}.")
+
+ size, rank = comm.size, comm.rank # Set up communication.
+
+ if size == 1: # Use torch functionality in non-parallel case.
+ return torch.cdist(x, y)
+
+ else: # Parallel case
+ # --- Setup and Matrix Initialization ---
+ mp, f = x.shape # Get number of samples in local chunk of x and number of features.
+ np = y.shape[0] # Get number of samples in local chunk of y.
+
+ # Each process initializes a local matrix, `local_distances`, of shape `(mp, n)`, where `mp` is the local chunk
+ # size of `x`, and `n` is the total number of samples in `y`. Each rank thus calculates the distance matrix
+ # chunk of size `mp x n`, i.e., rank 0 has distances from its own local `x` to all other `y`'s.
+
+ # Determine overall number of samples in y.
+ n = comm.allreduce(np, op=MPI.SUM)
+ print(f"Overall number of samples is {n}.")
+ # Initialize rank-local chunk of mp x n distance matrix with zeros.
+ local_distances = torch.zeros((mp, n))
+
+ # --- Managing Chunks and Displacements ---
+ # Determine where to put each result in the rank-local distance matrix chunk.
+ # Determine number of samples (rows) in each rank-local y.
+ y_counts = torch.tensor(comm.allgather(torch.numel(y) // f), dtype=torch.int)
+ # Calculate corresponding displacements from counts to record the starting index of each chunk in y. Thus, each
+ # process can identify where in the result matrix it should write the distances.
+ y_displ = (0,) + tuple(torch.cumsum(y_counts, dim=0, dtype=torch.int)[:-1])
+ # Extract actual result columns in distance matrix chunk for each rank.
+ cols = (y_displ[rank], y_displ[rank + 1] if (rank + 1) != size else n)
+
+ # --- Ring Communication Pattern ---
+ # Calculate distances in a "ring" pattern. Each process calculates distances for its local x chunk against its
+ # local y chunk (diagonal calculation). Then, through `size - 1` iterations, each process sends its y chunk to
+ # the next process in the "ring" while receiving a new y chunk from the previous process. This continues until
+ # each process has calculated distances between its x chunk and all chunks of y across all processes.
+ x_ = x
+ stationary = y
+
+ # 0th iteration: Calculate diagonal of global distance matrix.
+ # Each process calculates distances for its local x chunk against its local y chunk.
+ print(f"Rank [{rank}/{size}]: Calculate diagonal blocks...")
+ d_ij = torch.cdist(x_, stationary)
+ local_distances[:, cols[0]: cols[1]] = d_ij
+
+ print(f"Rank [{rank}/{size}]: Start tile-wise ring communication...")
+ # Remaining `(size+1) // 2` iterations: Send rank-local part of y to next process in circular fashion.
+ # We can perform less iterations due to the symmetric nature of the metric.
+ for iter_idx in range(1, (size + 2) // 2):
+ print(f"Rank [{rank}/{size}]: Starting iteration {iter_idx}")
+ # Since in the final iteration (with and even size) we only need to send from the first half of the
+ # processes, we check if this is the final iteration.
+ is_final_iteration = iter_idx == (size + 1) // 2
+ receiver = (rank + iter_idx) % size # Determine receiving process.
+ sender = (rank - iter_idx) % size # Determine sending process.
+ # Determine columns of rank-local distance matrix chunk to write result to.
+ col1 = y_displ[sender]
+ col2 = y_displ[sender + 1] if sender != size - 1 else n
+ columns = (col1, col2)
+
+ # Since the final iteration is different, we handle the communication differently.
+ if is_final_iteration:
+ # The second half of the processes only need to receive.
+ if rank >= size // 2:
+ stat = MPI.Status()
+ # Probe for incoming message containing the next chunk of y to consider.
+ comm.Probe(source=sender, tag=iter_idx, status=stat)
+ # Determine number of samples to receive (= overall number of floats to receive / number of features).
+ count = int(stat.Get_count(MPI.FLOAT) / f)
+ # Initialize tensor for incoming chunk of y with zeros.
+ moving = torch.zeros((count, f))
+ comm.Recv(moving, source=sender, tag=iter_idx)
+ # Calculate distances between stationary chunk of x and currently considered, moving chunk of y.
+ # Write result at correct position in distance matrix.
+ d_ij = torch.cdist(x_, moving)
+ local_distances[:, columns[0]: columns[1]] = d_ij
+ # The first half of the processes only need to send.
+ if rank < size // 2:
+ comm.Send(stationary, dest=receiver, tag=iter_idx)
+ # Standard communication for all but the final iteration.
+ else:
+ # All but first `iter_idx` processes are first receiving, then sending.
+ if (rank // iter_idx) != 0:
+ stat = MPI.Status()
+ # Probe for incoming message containing the next chunk of y to consider.
+ comm.Probe(source=sender, tag=iter_idx, status=stat)
+ # Determine number of samples to receive (= overall number of floats to receive / number of features).
+ count = int(stat.Get_count(MPI.FLOAT) / f)
+ # Initialize tensor for incoming chunk of y with zeros.
+ moving = torch.zeros((count, f))
+ comm.Recv(moving, source=sender, tag=iter_idx)
+ # Send rank-local chunk of y to next process.
+ comm.Send(stationary, dest=receiver, tag=iter_idx)
+ # First `iter_idx` processes can now receive after sending.
+ if (rank // iter_idx) == 0:
+ stat = MPI.Status()
+ comm.Probe(source=sender, tag=iter_idx, status=stat)
+ count = int(stat.Get_count(MPI.FLOAT) / f)
+ moving = torch.zeros((count, f))
+ comm.Recv(moving, source=sender, tag=iter_idx)
+ # Calculate distances between stationary chunk of x and currently considered, moving chunk of y.
+ # Write result at correct position in distance matrix.
+ d_ij = torch.cdist(x_, moving)
+ local_distances[:, columns[0]: columns[1]] = d_ij
+
+ # Step 2.3 - Only for symmetric metrics:
+ # We want to send the calculated distances back to process they came from since with symmetric metrics
+ # d(i,j) = d(j,i) and we can save calculation.
+ # First, we determine the columns of the distance matrix that the receiver will need to write.
+ symmetric_col1 = y_displ[receiver]
+ symmetric_col2 = y_displ[receiver + 1] if receiver != size - 1 else n
+ symmetric_columns = (symmetric_col1, symmetric_col2)
+
+ # Now, we send the calculated distances back to the symmetric tiles.
+ # We need to calculate the chunk size, therefore subtracting the smaller column index from the larger
+ # column index gives us the number of samples.
+ symmetric = torch.zeros(symmetric_columns[1] - symmetric_columns[0], (cols[1] - cols[0]))
+
+ # Communication for the final iteration.
+ if is_final_iteration:
+ # Now the first half of the processes are receiving the symmetrical values back.
+ if rank < size // 2:
+ comm.Recv(symmetric, source=receiver, tag=iter_idx)
+ # Save into the local distance matrix.
+ local_distances[:, symmetric_columns[0]: symmetric_columns[1]] = symmetric.transpose(0, 1)
+ # The second half that performed the calculation send the results back.
+ if rank >= size // 2:
+ comm.Send(d_ij, dest=sender, tag=iter_idx)
+ # Save the local distance matrix, nothing is received.
+ local_distances[:, symmetric_columns[0]: symmetric_columns[1]] = d_ij
+ # Standard communication handling.
+ else:
+ # As before, all but first `iter_idx` processes are first receiving, then sending.
+ if (rank // iter_idx) != 0:
+ comm.Recv(symmetric, source=receiver, tag=iter_idx)
+ # Then send.
+ comm.Send(d_ij, dest=sender, tag=iter_idx)
+ # Now the first `iter_idx` process can also receive after sending.
+ if (rank // iter_idx) == 0:
+ comm.Recv(symmetric, source=receiver, tag=iter_idx)
+ # Save into the local distance matrix.
+ local_distances[:, symmetric_columns[0]: symmetric_columns[1]] = symmetric.transpose(0, 1)
+ print(f"Rank [{rank}/{size}]: [DONE]")
+
+ return local_distances
+
+
+if __name__ == "__main__":
+ comm = MPI.COMM_WORLD
+ rank, size = comm.rank, comm.size
+
+ data_path = "./SUSY_50k.h5"
+ dataset = "data"
+
+ if rank == 0:
+ print(
+ "######################\n"
+ "# Pairwise distances #\n"
+ "######################\n"
+ f"COMM_WORLD size is {size}.\n"
+ f"Loading data... {data_path}[{dataset}]"
+ )
+
+ # Parallel data loader for SUSY data.
+ with h5py.File(data_path, "r") as handle:
+ chunk = int(handle[dataset].shape[0] / size)
+ if rank == size - 1:
+ data = torch.FloatTensor(handle[dataset][rank * chunk:])
+ else:
+ data = torch.FloatTensor(handle[dataset][rank * chunk:(rank + 1) * chunk])
+
+ print(f"\t[OK]\nRank [{rank}/{size}]: Local data chunk has shape {list(data.shape)}...")
+
+ if rank == 0:
+ print("Start distance calculations...")
+ # Calculate distances of all SUSY samples w.r.t. each other and measure runtime.
+ start = time.perf_counter()
+ distances = dist_symmetric(data, data, comm)
+ local_runtime = time.perf_counter() - start
+ # Calculate process-averaged runtime.
+ average_runtime = comm.allreduce(local_runtime, op=MPI.SUM) / size
+ print(f"Rank [{rank}/{size}]: Local distance matrix has shape {list(distances.shape)}.")
+ if rank == 0:
+ print(f"Process-averaged run time:\t{average_runtime} s")
diff --git a/2_psrs_cdist/solutions/A1/cdist_symmetric_weak.py b/2_psrs_cdist/solutions/A1/cdist_symmetric_weak.py
new file mode 100644
index 0000000..c96dc44
--- /dev/null
+++ b/2_psrs_cdist/solutions/A1/cdist_symmetric_weak.py
@@ -0,0 +1,226 @@
+import time
+
+import h5py
+import numpy as np
+import torch
+from mpi4py import MPI
+
+torch.set_default_dtype(torch.float32)
+
+
+def dist_symmetric(x: torch.Tensor, y: torch.Tensor, comm: MPI.Comm = MPI.COMM_WORLD) -> torch.Tensor:
+ """
+ Calculate pairwise distances between all rows (samples, i.e., along axis 0) of two tensors x and y in parallel.
+
+ The distance matrix is calculated tile-wise with ring communication between processes, each holding a piece of x
+ and/or y.
+
+ Parameters
+ ----------
+ x : torch.Tensor
+ First 2d tensor (of shape m/p x f). m is the total number of samples in x, distributed over p processors.
+ f is the number of features.
+ y : torch.Tensor
+ Second 2d tensor (of shape n/p x f). n is the total number of samples in x, distributed over p processors.
+ The number of features f must be the same as for x.
+ comm : MPI.Comm
+ Communicator to use. Default is ``MPI.COMM_WORLD``.
+ """
+ # Check whether two input tensors are compatible.
+ if len(x.shape) != len(y.shape) != 2:
+ raise ValueError("Input tensors must be two-dimensional.")
+ if x.shape[1] != y.shape[1]:
+ raise ValueError(f"Input tensors must have the same number of features but {x.shape[1]} != {y.shape[1]}.")
+
+ size, rank = comm.size, comm.rank # Set up communication.
+
+ if size == 1: # Use torch functionality in non-parallel case.
+ return torch.cdist(x, y)
+
+ else: # Parallel case
+ # --- Setup and Matrix Initialization ---
+ mp, f = x.shape # Get number of samples in local chunk of x and number of features.
+ np = y.shape[0] # Get number of samples in local chunk of y.
+
+ # Each process initializes a local matrix, `local_distances`, of shape `(mp, n)`, where `mp` is the local chunk
+ # size of `x`, and `n` is the total number of samples in `y`. Each rank thus calculates the distance matrix
+ # chunk of size `mp x n`, i.e., rank 0 has distances from its own local `x` to all other `y`'s.
+
+ # Determine overall number of samples in y.
+ n = comm.allreduce(np, op=MPI.SUM)
+ print(f"Overall number of samples is {n}.")
+ # Initialize rank-local chunk of mp x n distance matrix with zeros.
+ local_distances = torch.zeros((mp, n))
+
+ # --- Managing Chunks and Displacements ---
+ # Determine where to put each result in the rank-local distance matrix chunk.
+ # Determine number of samples (rows) in each rank-local y.
+ y_counts = torch.tensor(comm.allgather(torch.numel(y) // f), dtype=torch.int)
+ # Calculate corresponding displacements from counts to record the starting index of each chunk in y. Thus, each
+ # process can identify where in the result matrix it should write the distances.
+ y_displ = (0,) + tuple(torch.cumsum(y_counts, dim=0, dtype=torch.int)[:-1])
+ # Extract actual result columns in distance matrix chunk for each rank.
+ cols = (y_displ[rank], y_displ[rank + 1] if (rank + 1) != size else n)
+
+ # --- Ring Communication Pattern ---
+ # Calculate distances in a "ring" pattern. Each process calculates distances for its local x chunk against its
+ # local y chunk (diagonal calculation). Then, through `size - 1` iterations, each process sends its y chunk to
+ # the next process in the "ring" while receiving a new y chunk from the previous process. This continues until
+ # each process has calculated distances between its x chunk and all chunks of y across all processes.
+ x_ = x
+ stationary = y
+
+ # 0th iteration: Calculate diagonal of global distance matrix.
+ # Each process calculates distances for its local x chunk against its local y chunk.
+ print(f"Rank [{rank}/{size}]: Calculate diagonal blocks...")
+ d_ij = torch.cdist(x_, stationary)
+ local_distances[:, cols[0]: cols[1]] = d_ij
+
+ print(f"Rank [{rank}/{size}]: Start tile-wise ring communication...")
+ # Remaining `(size+1) // 2` iterations: Send rank-local part of y to next process in circular fashion.
+ # We can perform less iterations due to the symmetric nature of the metric.
+ for iter_idx in range(1, (size + 2) // 2):
+ print(f"Rank [{rank}/{size}]: Starting iteration {iter_idx}")
+ # Since in the final iteration (with and even size) we only need to send from the first half of the
+ # processes, we check if this is the final iteration.
+ is_final_iteration = iter_idx == (size + 1) // 2
+ receiver = (rank + iter_idx) % size # Determine receiving process.
+ sender = (rank - iter_idx) % size # Determine sending process.
+ # Determine columns of rank-local distance matrix chunk to write result to.
+ col1 = y_displ[sender]
+ col2 = y_displ[sender + 1] if sender != size - 1 else n
+ columns = (col1, col2)
+
+ # Since the final iteration is different, we handle the communication differently.
+ if is_final_iteration:
+ # The second half of the processes only need to receive.
+ if rank >= size // 2:
+ stat = MPI.Status()
+ # Probe for incoming message containing the next chunk of y to consider.
+ comm.Probe(source=sender, tag=iter_idx, status=stat)
+ # Determine number of samples to receive (= overall number of floats to receive / number of features).
+ count = int(stat.Get_count(MPI.FLOAT) / f)
+ # Initialize tensor for incoming chunk of y with zeros.
+ moving = torch.zeros((count, f))
+ comm.Recv(moving, source=sender, tag=iter_idx)
+ # Calculate distances between stationary chunk of x and currently considered, moving chunk of y.
+ # Write result at correct position in distance matrix.
+ d_ij = torch.cdist(x_, moving)
+ local_distances[:, columns[0]: columns[1]] = d_ij
+ # The first half of the processes only need to send.
+ if rank < size // 2:
+ comm.Send(stationary, dest=receiver, tag=iter_idx)
+ # Standard communication for all but the final iteration.
+ else:
+ # All but first `iter_idx` processes are first receiving, then sending.
+ if (rank // iter_idx) != 0:
+ stat = MPI.Status()
+ # Probe for incoming message containing the next chunk of y to consider.
+ comm.Probe(source=sender, tag=iter_idx, status=stat)
+ # Determine number of samples to receive (= overall number of floats to receive / number of features).
+ count = int(stat.Get_count(MPI.FLOAT) / f)
+ # Initialize tensor for incoming chunk of y with zeros.
+ moving = torch.zeros((count, f))
+ comm.Recv(moving, source=sender, tag=iter_idx)
+ # Send rank-local chunk of y to next process.
+ comm.Send(stationary, dest=receiver, tag=iter_idx)
+ # First `iter_idx` processes can now receive after sending.
+ if (rank // iter_idx) == 0:
+ stat = MPI.Status()
+ comm.Probe(source=sender, tag=iter_idx, status=stat)
+ count = int(stat.Get_count(MPI.FLOAT) / f)
+ moving = torch.zeros((count, f))
+ comm.Recv(moving, source=sender, tag=iter_idx)
+ # Calculate distances between stationary chunk of x and currently considered, moving chunk of y.
+ # Write result at correct position in distance matrix.
+ d_ij = torch.cdist(x_, moving)
+ local_distances[:, columns[0]: columns[1]] = d_ij
+
+ # Step 2.3 - Only for symmetric metrics:
+ # We want to send the calculated distances back to process they came from since with symmetric metrics
+ # d(i,j) = d(j,i) and we can save calculation.
+ # First, we determine the columns of the distance matrix that the receiver will need to write.
+ symmetric_col1 = y_displ[receiver]
+ symmetric_col2 = y_displ[receiver + 1] if receiver != size - 1 else n
+ symmetric_columns = (symmetric_col1, symmetric_col2)
+
+ # Now, we send the calculated distances back to the symmetric tiles.
+ # We need to calculate the chunk size, therefore subtracting the smaller column index from the larger
+ # column index gives us the number of samples.
+ symmetric = torch.zeros(symmetric_columns[1] - symmetric_columns[0], (cols[1] - cols[0]))
+
+ # Communication for the final iteration.
+ if is_final_iteration:
+ # Now the first half of the processes are receiving the symmetrical values back.
+ if rank < size // 2:
+ comm.Recv(symmetric, source=receiver, tag=iter_idx)
+ # Save into the local distance matrix.
+ local_distances[:, symmetric_columns[0]: symmetric_columns[1]] = symmetric.transpose(0, 1)
+ # The second half that performed the calculation send the results back.
+ if rank >= size // 2:
+ comm.Send(d_ij, dest=sender, tag=iter_idx)
+ # Save the local distance matrix, nothing is received.
+ local_distances[:, symmetric_columns[0]: symmetric_columns[1]] = d_ij
+ # Standard communication handling.
+ else:
+ # As before, all but first `iter_idx` processes are first receiving, then sending.
+ if (rank // iter_idx) != 0:
+ comm.Recv(symmetric, source=receiver, tag=iter_idx)
+ # Then send.
+ comm.Send(d_ij, dest=sender, tag=iter_idx)
+ # Now the first `iter_idx` process can also receive after sending.
+ if (rank // iter_idx) == 0:
+ comm.Recv(symmetric, source=receiver, tag=iter_idx)
+ # Save into the local distance matrix.
+ local_distances[:, symmetric_columns[0]: symmetric_columns[1]] = symmetric.transpose(0, 1)
+ print(f"Rank [{rank}/{size}]: [DONE]")
+
+ return local_distances
+
+
+if __name__ == "__main__":
+ comm = MPI.COMM_WORLD
+ rank, size = comm.rank, comm.size
+
+ data_path = "/pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5"
+ dataset = "data"
+
+ if rank == 0:
+ print(
+ "######################\n"
+ "# Pairwise distances #\n"
+ "######################\n"
+ f"COMM_WORLD size is {size}.\n"
+ f"Loading data... {data_path}[{dataset}]"
+ )
+
+ # Parallel data loader for SUSY data.
+ with h5py.File(data_path, "r") as handle:
+ nmax = handle[dataset].shape[0]
+ pmax = 16
+ n_samples = int(np.round(np.sqrt(nmax**2*size/pmax)))
+ print(
+ f"Overall number of samples considered: {n_samples}\n"
+ f"n^2/p = {n_samples**2/size}"
+ )
+ dset = handle[dataset][:n_samples]
+ chunk = int(dset.shape[0]/size)
+ if rank == size-1:
+ data = torch.FloatTensor(dset[rank*chunk:])
+ else:
+ data = torch.FloatTensor(dset[rank*chunk:(rank+1)*chunk])
+
+ print(f"\t[OK]\nRank [{rank}/{size}]: Local data chunk has shape {list(data.shape)}...")
+
+ if rank == 0:
+ print("Start distance calculations...")
+ # Calculate distances of all SUSY samples w.r.t. each other and measure runtime.
+ start = time.perf_counter()
+ distances = dist_symmetric(data, data, comm)
+ local_runtime = time.perf_counter() - start
+ # Calculate process-averaged runtime.
+ average_runtime = comm.allreduce(local_runtime, op=MPI.SUM) / size
+ print(f"Rank [{rank}/{size}]: Local distance matrix has shape {list(distances.shape)}.")
+ if rank == 0:
+ print(f"Process-averaged run time:\t{average_runtime} s")
+
diff --git a/2_psrs_cdist/solutions/A1/cdist_weak.py b/2_psrs_cdist/solutions/A1/cdist_weak.py
new file mode 100644
index 0000000..bb05b55
--- /dev/null
+++ b/2_psrs_cdist/solutions/A1/cdist_weak.py
@@ -0,0 +1,159 @@
+"""Parallel calculation of pairwise distances"""
+import time
+
+import h5py
+import numpy as np
+import torch
+from mpi4py import MPI
+
+torch.set_default_dtype(torch.float32)
+
+
+def dist(x: torch.Tensor, y: torch.Tensor, comm: MPI.Comm = MPI.COMM_WORLD) -> torch.Tensor:
+ """
+ Calculate pairwise distances between all rows (samples, i.e., along axis 0) of two tensors x and y in parallel.
+
+ The distance matrix is calculated tile-wise with ring communication between processes, each holding a piece of x
+ and/or y.
+
+ Parameters
+ ----------
+ x : torch.Tensor
+ First 2d tensor (of shape m/p x f). m is the total number of samples in x, distributed over p processors.
+ f is the number of features.
+ y : torch.Tensor
+ Second 2d tensor (of shape n/p x f). n is the total number of samples in x, distributed over p processors.
+ The number of features f must be the same as for x.
+ comm : MPI.Comm
+ Communicator to use. Default is ``MPI.COMM_WORLD``.
+ """
+ # Check whether two input tensors are compatible.
+ if len(x.shape) != len(y.shape) != 2:
+ raise ValueError("Input tensors must be two-dimensional.")
+ if x.shape[1] != y.shape[1]:
+ raise ValueError(f"Input tensors must have the same number of features but {x.shape[1]} != {y.shape[1]}.")
+
+ size, rank = comm.size, comm.rank # Set up communication.
+
+ if size == 1: # Use torch functionality in non-parallel case.
+ return torch.cdist(x, y)
+
+ else: # Parallel case
+ # --- Setup and Matrix Initialization ---
+ mp, f = x.shape # Get number of samples in local chunk of x and number of features.
+ np = y.shape[0] # Get number of samples in local chunk of y.
+
+ # Each process initializes a local matrix, `local_distances`, of shape `(mp, n)`, where `mp` is the local chunk
+ # size of `x`, and `n` is the total number of samples in `y`. Each rank thus calculates the distance matrix
+ # chunk of size `mp x n`, i.e., rank 0 has distances from its own local `x` to all other `y`'s.
+
+ # Determine overall number of samples in y.
+ n = comm.allreduce(np, op=MPI.SUM)
+ print(f"Overall number of samples is {n}.")
+ # Initialize rank-local chunk of mp x n distance matrix with zeros.
+ local_distances = torch.zeros((mp, n))
+
+ # --- Managing Chunks and Displacements ---
+ # Determine where to put each result in the rank-local distance matrix chunk.
+ # Determine number of samples (rows) in each rank-local y.
+ y_counts = torch.tensor(comm.allgather(np), dtype=torch.int)
+ # Calculate corresponding displacements from counts to record the starting index of each chunk in y. Thus, each
+ # process can identify where in the result matrix it should write the distances.
+ y_displ = (0,) + tuple(torch.cumsum(y_counts, dim=0, dtype=torch.int)[:-1])
+
+ # --- Ring Communication Pattern ---
+ # Calculate distances in a "ring" pattern. Each process calculates distances for its local x chunk against its
+ # local y chunk (diagonal calculation). Then, through `size - 1` iterations, each process sends its y chunk to
+ # the next process in the "ring" while receiving a new y chunk from the previous process. This continues until
+ # each process has calculated distances between its x chunk and all chunks of y across all processes.
+
+ # 0th iteration: Calculate diagonal of global distance matrix.
+ # Each process calculates distances for its local x chunk against its local y chunk.
+ print(f"Rank [{rank}/{size}]: Calculate diagonal blocks in global distance matrix...")
+ # Extract actual result columns in distance matrix chunk for each rank.
+ cols = (y_displ[rank], y_displ[rank + 1] if (rank + 1) != size else n)
+ d_ij = torch.cdist(x, y)
+ local_distances[:, cols[0]: cols[1]] = d_ij
+
+ print(f"Rank [{rank}/{size}]: Start tile-wise ring communication...")
+ # Remaining `size-1` iterations: Send rank-local part of y to next process in circular fashion.
+ for iter_idx in range(1, size):
+
+ receiver = (rank + iter_idx) % size # Determine receiving process.
+ sender = (rank - iter_idx) % size # Determine sending process.
+ # Determine columns of rank-local distance matrix chunk to write result to.
+ col1 = y_displ[sender]
+ col2 = y_displ[sender + 1] if sender != size - 1 else n
+ # All but first `iter_idx` processes are first receiving, then sending.
+ if (rank // iter_idx) != 0:
+ stat = MPI.Status()
+ # Probe for incoming message containing the next chunk of y to consider.
+ comm.Probe(source=sender, tag=iter_idx, status=stat)
+ # Determine number of samples to receive (= overall number of floats to receive / number of features).
+ count = int(stat.Get_count(MPI.FLOAT) / f)
+ # Initialize tensor for incoming chunk of y with zeros.
+ moving = torch.zeros((count, f))
+ comm.Recv(moving, source=sender, tag=iter_idx)
+ # Send rank-local chunk of y to next process.
+ comm.Send(y, dest=receiver, tag=iter_idx)
+ # First `iter_idx` processes can now receive after sending.
+ if (rank // iter_idx) == 0:
+ stat = MPI.Status()
+ comm.Probe(source=sender, tag=iter_idx, status=stat)
+ count = int(stat.Get_count(MPI.FLOAT) / f)
+ moving = torch.zeros((count, f))
+ comm.Recv(moving, source=sender, tag=iter_idx)
+ # Calculate distances between stationary chunk of x and currently considered, moving chunk of y.
+ # Write result at correct position in distance matrix.
+ d_ij = torch.cdist(x, moving)
+ local_distances[:, col1: col2] = d_ij
+ print(f"Rank [{rank}/{size}]: [DONE]")
+
+ return local_distances
+
+
+if __name__ == "__main__":
+ comm = MPI.COMM_WORLD
+ rank, size = comm.rank, comm.size
+
+ data_path = "/pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5"
+ dataset = "data"
+
+ if rank == 0:
+ print(
+ "######################\n"
+ "# Pairwise distances #\n"
+ "######################\n"
+ f"COMM_WORLD size is {size}.\n"
+ f"Loading data... {data_path}[{dataset}]"
+ )
+
+ # Parallel data loader for SUSY data.
+ with h5py.File(data_path, "r") as handle:
+ nmax = handle[dataset].shape[0]
+ pmax = 16
+ n_samples = int(np.round(np.sqrt(nmax**2*size/pmax)))
+ print(
+ f"Overall number of samples considered: {n_samples}\n"
+ f"n^2/p = {n_samples**2/size}"
+ )
+ dset = handle[dataset][:n_samples]
+ chunk = int(dset.shape[0]/size)
+ if rank == size-1:
+ data = torch.FloatTensor(dset[rank*chunk:])
+ else:
+ data = torch.FloatTensor(dset[rank*chunk:(rank+1)*chunk])
+
+ print(f"\t[OK]\nRank [{rank}/{size}]: Local data chunk has shape {list(data.shape)}...")
+
+ if rank == 0:
+ print("Start distance calculations...")
+ # Calculate distances of all SUSY samples w.r.t. each other and measure runtime.
+ start = time.perf_counter()
+ distances = dist(data, data, comm)
+ local_runtime = time.perf_counter() - start
+ # Calculate process-averaged runtime.
+ average_runtime = comm.allreduce(local_runtime, op=MPI.SUM) / size
+ print(f"Rank [{rank}/{size}]: Local distance matrix has shape {list(distances.shape)}.")
+ if rank == 0:
+ print(f"Process-averaged run time:\t{average_runtime} s")
diff --git a/2_psrs_cdist/solutions/A1/plot_scaling.py b/2_psrs_cdist/solutions/A1/plot_scaling.py
new file mode 100644
index 0000000..ff3cf26
--- /dev/null
+++ b/2_psrs_cdist/solutions/A1/plot_scaling.py
@@ -0,0 +1,172 @@
+import argparse
+import os
+import pathlib
+import re
+from glob import glob
+from typing import Optional
+
+import matplotlib.pyplot as plt
+
+
+def process_file(file_path: str) -> Optional[float]:
+ """
+ Extract all process-averaged run times, remove the largest value, and compute the average.
+
+ Parameters
+ ----------
+ file_path : str
+ Path to the .out file.
+
+ Returns
+ -------
+ Optional[float]
+ The average of the run times after dropping the largest value, or None if there
+ are not enough values.
+ """
+ runtime_pattern = r"Process-averaged run time:\s+(\d+\.\d+)"
+
+ with open(file_path, "r") as file:
+ # Extract all run time values as floats.
+ times = [float(match) for match in re.findall(runtime_pattern, file.read())]
+
+ # If fewer than two values are found, return None (not enough data to drop the highest value)
+ if len(times) <= 1:
+ return
+
+ # Drop the largest value.
+ times.remove(max(times))
+
+ # Calculate and return the average of the remaining values.
+ return sum(times) / len(times)
+
+
+def extract_node_count(filename: str) -> Optional[int]:
+ """
+ Extract the node count from a filename.
+
+ Parameters
+ ----------
+ filename : str
+ The name of the .out file.
+
+ Returns
+ -------
+ Optional[int]
+ The extracted node count as an integer, or None if not found.
+ """
+ filename_pattern = r"nodes_(\d+)\.out"
+ match = re.search(filename_pattern, filename)
+ return int(match.group(1)) if match else None
+
+
+def process_out_files(folder_path: str) -> dict[int, Optional[float]]:
+ """
+ Processes all .out files in a folder to calculate average run times by node count.
+
+ Parameters
+ ----------
+ folder_path : str
+ Path to the folder containing .out files.
+
+ Returns
+ -------
+ dict[int, Optional[float]]
+ Dictionary mapping each node count to the average run time (or None if insufficient data).
+ """
+ results = {}
+ for file_path in glob(os.path.join(folder_path, "*.out")):
+ filename = os.path.basename(file_path)
+ node_count = extract_node_count(filename)
+
+ if node_count is not None:
+ avg_time = process_file(file_path)
+ if avg_time is not None:
+ results[node_count] = avg_time
+
+ # Display results
+ for nodes, avg_time in sorted(results.items()):
+ print(f"Nodes {nodes}: Average time (without max value) = {avg_time:.5f} s")
+
+ return results
+
+
+def plot_results(results: dict[int, float], scaling_type: str, file_path: pathlib.Path | str) -> None:
+ """
+ Plot runtime, speedup, and efficiency based on the results dictionary.
+
+ Parameters
+ ----------
+ results : dict[int, float]
+ Dictionary containing average run time for each node count.
+ scaling_type : str
+ The type of scaling considered. Must be either "strong" or "weak".
+ """
+ # Sort results by number of nodes.
+ nodes = sorted(results.keys())
+ runtimes = [results[n] for n in nodes]
+
+ if scaling_type == "strong":
+ # Calculate speedup (using the runtime with the fewest nodes as the baseline)
+ baseline_runtime = runtimes[0]
+ derived_quantity = [baseline_runtime / runtime for runtime in runtimes]
+ derived_label = "Speedup"
+ ideal = nodes
+ elif scaling_type == "weak":
+ baseline_runtime = runtimes[0]
+ derived_quantity = [baseline_runtime / runtime for runtime in runtimes]
+ derived_label = "Efficiency"
+ ideal = len(nodes) * [1]
+ y_lim = [0, 1.1]
+ else:
+ raise ValueError(f"Unknown scaling type: {scaling_type}")
+ print(derived_quantity)
+
+ # Settings
+ labelsize = "x-large"
+
+ # Plotting
+ fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 5))
+
+ # Runtime vs Number of Nodes
+ ax1.set_facecolor("lightgrey")
+ ax1.plot(nodes, runtimes, marker="X", ms=10, color="#009682")
+ ax1.set_xlabel("Number of nodes", fontweight="bold", fontsize=labelsize)
+ ax1.set_ylabel("Runtime / s", fontweight="bold", fontsize=labelsize)
+ ax1.grid(True)
+ ax1.grid(color="white", linewidth=1.7)
+ ax1.tick_params(axis="both", labelsize=labelsize)
+ ax1.set_xticks(nodes, [str(node) for node in nodes])
+
+ # Speedup / Efficiency
+ ax2.set_facecolor("lightgrey")
+ ax2.plot(nodes, ideal, ls="--", color="k", label="Ideal")
+ ax2.plot(nodes, derived_quantity, marker="X", ms=10, color="#009682", label="Actual")
+ ax2.set_xlabel("Number of nodes", fontweight="bold", fontsize=labelsize)
+ ax2.set_ylabel(derived_label, fontweight="bold", fontsize=labelsize)
+ ax2.legend()
+ ax2.grid(True)
+ ax2.grid(color="white", linewidth=1.7)
+ ax2.tick_params(axis="both", labelsize=labelsize)
+ ax2.set_xticks(nodes, [str(node) for node in nodes])
+ if scaling_type == "weak":
+ ax2.set_ylim(y_lim)
+
+ plt.tight_layout()
+ plt.savefig(pathlib.Path(file_path) / f"{file_path}.pdf")
+ plt.show()
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(
+ prog="Plot cdist scaling results",
+ )
+ parser.add_argument( # Function to optimize
+ "--path",
+ type=pathlib.Path,
+ default="./",
+ )
+ config = parser.parse_args()
+ # Directory containing the .out files
+ results = process_out_files(config.path)
+ # Plot the results
+ plot_results(results, scaling_type=str(config.path).split("_")[0], file_path=pathlib.Path(config.path))
diff --git a/2_psrs_cdist/solutions/A1/strong_scaling/nodes_1.out b/2_psrs_cdist/solutions/A1/strong_scaling/nodes_1.out
new file mode 100644
index 0000000..7aef334
--- /dev/null
+++ b/2_psrs_cdist/solutions/A1/strong_scaling/nodes_1.out
@@ -0,0 +1,136 @@
+Run 1
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/1]: Local data chunk has shape [50000, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [50000, 50000].
+Process-averaged run time: 6.33100987970829 s
+Run 2
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/1]: Local data chunk has shape [50000, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [50000, 50000].
+Process-averaged run time: 4.461751670576632 s
+Run 3
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/1]: Local data chunk has shape [50000, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [50000, 50000].
+Process-averaged run time: 4.44517351873219 s
+Run 4
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/1]: Local data chunk has shape [50000, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [50000, 50000].
+Process-averaged run time: 4.454093327745795 s
+Run 5
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/1]: Local data chunk has shape [50000, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [50000, 50000].
+Process-averaged run time: 4.474684880115092 s
+Run 6
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/1]: Local data chunk has shape [50000, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [50000, 50000].
+Process-averaged run time: 4.447230387479067 s
+Run 7
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/1]: Local data chunk has shape [50000, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [50000, 50000].
+Process-averaged run time: 4.445676466450095 s
+Run 8
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/1]: Local data chunk has shape [50000, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [50000, 50000].
+Process-averaged run time: 4.4390754187479615 s
+Run 9
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/1]: Local data chunk has shape [50000, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [50000, 50000].
+Process-averaged run time: 4.455759993754327 s
+Run 10
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/1]: Local data chunk has shape [50000, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [50000, 50000].
+Process-averaged run time: 4.4520881017670035 s
+Run 11
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/1]: Local data chunk has shape [50000, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [50000, 50000].
+Process-averaged run time: 4.454913998953998 s
+
+============================= JOB FEEDBACK =============================
+
+NodeName=uc2n420
+Job ID: 24674001
+Cluster: uc2
+User/Group: ku4408/scc
+State: COMPLETED (exit code 0)
+Nodes: 1
+Cores per node: 40
+CPU Utilized: 00:01:13
+CPU Efficiency: 1.12% of 01:48:40 core-walltime
+Job Wall-clock time: 00:02:43
+Memory Utilized: 9.81 GB
+Memory Efficiency: 22.31% of 43.95 GB
diff --git a/2_psrs_cdist/solutions/A1/strong_scaling/nodes_16.out b/2_psrs_cdist/solutions/A1/strong_scaling/nodes_16.out
new file mode 100644
index 0000000..96a2d1f
--- /dev/null
+++ b/2_psrs_cdist/solutions/A1/strong_scaling/nodes_16.out
@@ -0,0 +1,1335 @@
+Run 1
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [10/16]: Calculate diagonal blocks in global distance matrix...
+Rank [6/16]: Calculate diagonal blocks in global distance matrix...
+Rank [13/16]: Calculate diagonal blocks in global distance matrix...
+Rank [11/16]: Calculate diagonal blocks in global distance matrix...
+Rank [9/16]: Calculate diagonal blocks in global distance matrix...
+Rank [14/16]: Calculate diagonal blocks in global distance matrix...
+Rank [1/16]: Calculate diagonal blocks in global distance matrix...
+Rank [4/16]: Calculate diagonal blocks in global distance matrix...
+Rank [3/16]: Calculate diagonal blocks in global distance matrix...
+Rank [15/16]: Calculate diagonal blocks in global distance matrix...
+Rank [5/16]: Calculate diagonal blocks in global distance matrix...
+Rank [7/16]: Calculate diagonal blocks in global distance matrix...
+Rank [2/16]: Calculate diagonal blocks in global distance matrix...
+Rank [0/16]: Calculate diagonal blocks in global distance matrix...
+Rank [12/16]: Calculate diagonal blocks in global distance matrix...
+Rank [8/16]: Calculate diagonal blocks in global distance matrix...
+Rank [10/16]: Start tile-wise ring communication...
+Rank [14/16]: Start tile-wise ring communication...
+Rank [6/16]: Start tile-wise ring communication...
+Rank [5/16]: Start tile-wise ring communication...
+Rank [13/16]: Start tile-wise ring communication...
+Rank [3/16]: Start tile-wise ring communication...
+Rank [11/16]: Start tile-wise ring communication...
+Rank [15/16]: Start tile-wise ring communication...
+Rank [9/16]: Start tile-wise ring communication...
+Rank [1/16]: Start tile-wise ring communication...
+Rank [4/16]: Start tile-wise ring communication...
+Rank [7/16]: Start tile-wise ring communication...
+Rank [2/16]: Start tile-wise ring communication...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [12/16]: Start tile-wise ring communication...
+Rank [8/16]: Start tile-wise ring communication...
+Rank [15/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.8350668167695403 s
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Run 2
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks in global distance matrix...
+Rank [3/16]: Calculate diagonal blocks in global distance matrix...
+Rank [13/16]: Calculate diagonal blocks in global distance matrix...
+Rank [7/16]: Calculate diagonal blocks in global distance matrix...
+Rank [4/16]: Calculate diagonal blocks in global distance matrix...
+Rank [11/16]: Calculate diagonal blocks in global distance matrix...
+Rank [2/16]: Calculate diagonal blocks in global distance matrix...
+Rank [14/16]: Calculate diagonal blocks in global distance matrix...
+Rank [15/16]: Calculate diagonal blocks in global distance matrix...
+Rank [5/16]: Calculate diagonal blocks in global distance matrix...
+Rank [12/16]: Calculate diagonal blocks in global distance matrix...
+Rank [1/16]: Calculate diagonal blocks in global distance matrix...
+Rank [9/16]: Calculate diagonal blocks in global distance matrix...
+Rank [8/16]: Calculate diagonal blocks in global distance matrix...
+Rank [10/16]: Calculate diagonal blocks in global distance matrix...
+Rank [6/16]: Calculate diagonal blocks in global distance matrix...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [11/16]: Start tile-wise ring communication...
+Rank [13/16]: Start tile-wise ring communication...
+Rank [1/16]: Start tile-wise ring communication...
+Rank [9/16]: Start tile-wise ring communication...
+Rank [8/16]: Start tile-wise ring communication...
+Rank [4/16]: Start tile-wise ring communication...
+Rank [2/16]: Start tile-wise ring communication...
+Rank [5/16]: Start tile-wise ring communication...
+Rank [7/16]: Start tile-wise ring communication...
+Rank [3/16]: Start tile-wise ring communication...
+Rank [12/16]: Start tile-wise ring communication...
+Rank [14/16]: Start tile-wise ring communication...
+Rank [6/16]: Start tile-wise ring communication...
+Rank [10/16]: Start tile-wise ring communication...
+Rank [15/16]: Start tile-wise ring communication...
+Rank [15/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.6093131661764346 s
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Run 3
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks in global distance matrix...
+Rank [8/16]: Calculate diagonal blocks in global distance matrix...
+Rank [10/16]: Calculate diagonal blocks in global distance matrix...
+Rank [5/16]: Calculate diagonal blocks in global distance matrix...
+Rank [12/16]: Calculate diagonal blocks in global distance matrix...
+Rank [6/16]: Calculate diagonal blocks in global distance matrix...
+Rank [3/16]: Calculate diagonal blocks in global distance matrix...
+Rank [1/16]: Calculate diagonal blocks in global distance matrix...
+Rank [2/16]: Calculate diagonal blocks in global distance matrix...
+Rank [7/16]: Calculate diagonal blocks in global distance matrix...
+Rank [14/16]: Calculate diagonal blocks in global distance matrix...
+Rank [4/16]: Calculate diagonal blocks in global distance matrix...
+Rank [11/16]: Calculate diagonal blocks in global distance matrix...
+Rank [13/16]: Calculate diagonal blocks in global distance matrix...
+Rank [15/16]: Calculate diagonal blocks in global distance matrix...
+Rank [9/16]: Calculate diagonal blocks in global distance matrix...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [13/16]: Start tile-wise ring communication...
+Rank [8/16]: Start tile-wise ring communication...
+Rank [3/16]: Start tile-wise ring communication...
+Rank [2/16]: Start tile-wise ring communication...
+Rank [14/16]: Start tile-wise ring communication...
+Rank [5/16]: Start tile-wise ring communication...
+Rank [11/16]: Start tile-wise ring communication...
+Rank [9/16]: Start tile-wise ring communication...
+Rank [12/16]: Start tile-wise ring communication...
+Rank [6/16]: Start tile-wise ring communication...
+Rank [1/16]: Start tile-wise ring communication...
+Rank [7/16]: Start tile-wise ring communication...
+Rank [4/16]: Start tile-wise ring communication...
+Rank [15/16]: Start tile-wise ring communication...
+Rank [10/16]: Start tile-wise ring communication...
+Rank [15/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.5940211796551012 s
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Run 4
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks in global distance matrix...
+Rank [3/16]: Calculate diagonal blocks in global distance matrix...
+Rank [10/16]: Calculate diagonal blocks in global distance matrix...
+Rank [14/16]: Calculate diagonal blocks in global distance matrix...
+Rank [7/16]: Calculate diagonal blocks in global distance matrix...
+Rank [1/16]: Calculate diagonal blocks in global distance matrix...
+Rank [4/16]: Calculate diagonal blocks in global distance matrix...
+Rank [6/16]: Calculate diagonal blocks in global distance matrix...
+Rank [5/16]: Calculate diagonal blocks in global distance matrix...
+Rank [15/16]: Calculate diagonal blocks in global distance matrix...
+Rank [12/16]: Calculate diagonal blocks in global distance matrix...
+Rank [8/16]: Calculate diagonal blocks in global distance matrix...
+Rank [9/16]: Calculate diagonal blocks in global distance matrix...
+Rank [2/16]: Calculate diagonal blocks in global distance matrix...
+Rank [13/16]: Calculate diagonal blocks in global distance matrix...
+Rank [11/16]: Calculate diagonal blocks in global distance matrix...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [6/16]: Start tile-wise ring communication...
+Rank [8/16]: Start tile-wise ring communication...
+Rank [9/16]: Start tile-wise ring communication...
+Rank [13/16]: Start tile-wise ring communication...
+Rank [3/16]: Start tile-wise ring communication...
+Rank [2/16]: Start tile-wise ring communication...
+Rank [11/16]: Start tile-wise ring communication...
+Rank [10/16]: Start tile-wise ring communication...
+Rank [5/16]: Start tile-wise ring communication...
+Rank [7/16]: Start tile-wise ring communication...
+Rank [1/16]: Start tile-wise ring communication...
+Rank [4/16]: Start tile-wise ring communication...
+Rank [12/16]: Start tile-wise ring communication...
+Rank [14/16]: Start tile-wise ring communication...
+Rank [15/16]: Start tile-wise ring communication...
+Rank [15/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.6216935422853567 s
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Run 5
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks in global distance matrix...
+Rank [3/16]: Calculate diagonal blocks in global distance matrix...
+Rank [13/16]: Calculate diagonal blocks in global distance matrix...
+Rank [7/16]: Calculate diagonal blocks in global distance matrix...
+Rank [8/16]: Calculate diagonal blocks in global distance matrix...
+Rank [15/16]: Calculate diagonal blocks in global distance matrix...
+Rank [1/16]: Calculate diagonal blocks in global distance matrix...
+Rank [2/16]: Calculate diagonal blocks in global distance matrix...
+Rank [6/16]: Calculate diagonal blocks in global distance matrix...
+Rank [14/16]: Calculate diagonal blocks in global distance matrix...
+Rank [5/16]: Calculate diagonal blocks in global distance matrix...
+Rank [10/16]: Calculate diagonal blocks in global distance matrix...
+Rank [11/16]: Calculate diagonal blocks in global distance matrix...
+Rank [4/16]: Calculate diagonal blocks in global distance matrix...
+Rank [12/16]: Calculate diagonal blocks in global distance matrix...
+Rank [9/16]: Calculate diagonal blocks in global distance matrix...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [13/16]: Start tile-wise ring communication...
+Rank [6/16]: Start tile-wise ring communication...
+Rank [10/16]: Start tile-wise ring communication...
+Rank [8/16]: Start tile-wise ring communication...
+Rank [12/16]: Start tile-wise ring communication...
+Rank [9/16]: Start tile-wise ring communication...
+Rank [1/16]: Start tile-wise ring communication...
+Rank [5/16]: Start tile-wise ring communication...
+Rank [11/16]: Start tile-wise ring communication...
+Rank [4/16]: Start tile-wise ring communication...
+Rank [3/16]: Start tile-wise ring communication...
+Rank [2/16]: Start tile-wise ring communication...
+Rank [14/16]: Start tile-wise ring communication...
+Rank [7/16]: Start tile-wise ring communication...
+Rank [15/16]: Start tile-wise ring communication...
+Rank [15/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.6184260346344672 s
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: [DONE]
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Run 6
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks in global distance matrix...
+Rank [4/16]: Calculate diagonal blocks in global distance matrix...
+Rank [9/16]: Calculate diagonal blocks in global distance matrix...
+Rank [2/16]: Calculate diagonal blocks in global distance matrix...
+Rank [3/16]: Calculate diagonal blocks in global distance matrix...
+Rank [10/16]: Calculate diagonal blocks in global distance matrix...
+Rank [5/16]: Calculate diagonal blocks in global distance matrix...
+Rank [1/16]: Calculate diagonal blocks in global distance matrix...
+Rank [7/16]: Calculate diagonal blocks in global distance matrix...
+Rank [13/16]: Calculate diagonal blocks in global distance matrix...
+Rank [15/16]: Calculate diagonal blocks in global distance matrix...
+Rank [12/16]: Calculate diagonal blocks in global distance matrix...
+Rank [11/16]: Calculate diagonal blocks in global distance matrix...
+Rank [6/16]: Calculate diagonal blocks in global distance matrix...
+Rank [8/16]: Calculate diagonal blocks in global distance matrix...
+Rank [14/16]: Calculate diagonal blocks in global distance matrix...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [9/16]: Start tile-wise ring communication...
+Rank [13/16]: Start tile-wise ring communication...
+Rank [4/16]: Start tile-wise ring communication...
+Rank [1/16]: Start tile-wise ring communication...
+Rank [8/16]: Start tile-wise ring communication...
+Rank [3/16]: Start tile-wise ring communication...
+Rank [12/16]: Start tile-wise ring communication...
+Rank [5/16]: Start tile-wise ring communication...
+Rank [11/16]: Start tile-wise ring communication...
+Rank [6/16]: Start tile-wise ring communication...
+Rank [7/16]: Start tile-wise ring communication...
+Rank [15/16]: Start tile-wise ring communication...
+Rank [10/16]: Start tile-wise ring communication...
+Rank [14/16]: Start tile-wise ring communication...
+Rank [2/16]: Start tile-wise ring communication...
+Rank [15/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.6189388468046673 s
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Run 7
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks in global distance matrix...
+Rank [10/16]: Calculate diagonal blocks in global distance matrix...
+Rank [6/16]: Calculate diagonal blocks in global distance matrix...
+Rank [11/16]: Calculate diagonal blocks in global distance matrix...
+Rank [8/16]: Calculate diagonal blocks in global distance matrix...
+Rank [15/16]: Calculate diagonal blocks in global distance matrix...
+Rank [9/16]: Calculate diagonal blocks in global distance matrix...
+Rank [13/16]: Calculate diagonal blocks in global distance matrix...
+Rank [3/16]: Calculate diagonal blocks in global distance matrix...
+Rank [4/16]: Calculate diagonal blocks in global distance matrix...
+Rank [12/16]: Calculate diagonal blocks in global distance matrix...
+Rank [5/16]: Calculate diagonal blocks in global distance matrix...
+Rank [14/16]: Calculate diagonal blocks in global distance matrix...
+Rank [1/16]: Calculate diagonal blocks in global distance matrix...
+Rank [2/16]: Calculate diagonal blocks in global distance matrix...
+Rank [7/16]: Calculate diagonal blocks in global distance matrix...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [10/16]: Start tile-wise ring communication...
+Rank [9/16]: Start tile-wise ring communication...
+Rank [8/16]: Start tile-wise ring communication...
+Rank [13/16]: Start tile-wise ring communication...
+Rank [1/16]: Start tile-wise ring communication...
+Rank [2/16]: Start tile-wise ring communication...
+Rank [7/16]: Start tile-wise ring communication...
+Rank [11/16]: Start tile-wise ring communication...
+Rank [3/16]: Start tile-wise ring communication...
+Rank [4/16]: Start tile-wise ring communication...
+Rank [12/16]: Start tile-wise ring communication...
+Rank [5/16]: Start tile-wise ring communication...
+Rank [14/16]: Start tile-wise ring communication...
+Rank [6/16]: Start tile-wise ring communication...
+Rank [15/16]: Start tile-wise ring communication...
+Rank [15/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.6224825331592001 s
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Run 8
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks in global distance matrix...
+Rank [8/16]: Calculate diagonal blocks in global distance matrix...
+Rank [2/16]: Calculate diagonal blocks in global distance matrix...
+Rank [7/16]: Calculate diagonal blocks in global distance matrix...
+Rank [5/16]: Calculate diagonal blocks in global distance matrix...
+Rank [4/16]: Calculate diagonal blocks in global distance matrix...
+Rank [13/16]: Calculate diagonal blocks in global distance matrix...
+Rank [1/16]: Calculate diagonal blocks in global distance matrix...
+Rank [11/16]: Calculate diagonal blocks in global distance matrix...
+Rank [10/16]: Calculate diagonal blocks in global distance matrix...
+Rank [12/16]: Calculate diagonal blocks in global distance matrix...
+Rank [9/16]: Calculate diagonal blocks in global distance matrix...
+Rank [3/16]: Calculate diagonal blocks in global distance matrix...
+Rank [6/16]: Calculate diagonal blocks in global distance matrix...
+Rank [15/16]: Calculate diagonal blocks in global distance matrix...
+Rank [14/16]: Calculate diagonal blocks in global distance matrix...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [13/16]: Start tile-wise ring communication...
+Rank [2/16]: Start tile-wise ring communication...
+Rank [4/16]: Start tile-wise ring communication...
+Rank [1/16]: Start tile-wise ring communication...
+Rank [14/16]: Start tile-wise ring communication...
+Rank [5/16]: Start tile-wise ring communication...
+Rank [7/16]: Start tile-wise ring communication...
+Rank [11/16]: Start tile-wise ring communication...
+Rank [9/16]: Start tile-wise ring communication...
+Rank [3/16]: Start tile-wise ring communication...
+Rank [8/16]: Start tile-wise ring communication...
+Rank [6/16]: Start tile-wise ring communication...
+Rank [10/16]: Start tile-wise ring communication...
+Rank [12/16]: Start tile-wise ring communication...
+Rank [15/16]: Start tile-wise ring communication...
+Rank [15/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.6286917179822922 s
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Run 9
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks in global distance matrix...
+Rank [8/16]: Calculate diagonal blocks in global distance matrix...
+Rank [1/16]: Calculate diagonal blocks in global distance matrix...
+Rank [9/16]: Calculate diagonal blocks in global distance matrix...
+Rank [3/16]: Calculate diagonal blocks in global distance matrix...
+Rank [14/16]: Calculate diagonal blocks in global distance matrix...
+Rank [10/16]: Calculate diagonal blocks in global distance matrix...
+Rank [2/16]: Calculate diagonal blocks in global distance matrix...
+Rank [11/16]: Calculate diagonal blocks in global distance matrix...
+Rank [12/16]: Calculate diagonal blocks in global distance matrix...
+Rank [7/16]: Calculate diagonal blocks in global distance matrix...
+Rank [13/16]: Calculate diagonal blocks in global distance matrix...
+Rank [15/16]: Calculate diagonal blocks in global distance matrix...
+Rank [6/16]: Calculate diagonal blocks in global distance matrix...
+Rank [4/16]: Calculate diagonal blocks in global distance matrix...
+Rank [5/16]: Calculate diagonal blocks in global distance matrix...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [8/16]: Start tile-wise ring communication...
+Rank [10/16]: Start tile-wise ring communication...
+Rank [13/16]: Start tile-wise ring communication...
+Rank [4/16]: Start tile-wise ring communication...
+Rank [12/16]: Start tile-wise ring communication...
+Rank [1/16]: Start tile-wise ring communication...
+Rank [6/16]: Start tile-wise ring communication...
+Rank [5/16]: Start tile-wise ring communication...
+Rank [3/16]: Start tile-wise ring communication...
+Rank [2/16]: Start tile-wise ring communication...
+Rank [11/16]: Start tile-wise ring communication...
+Rank [7/16]: Start tile-wise ring communication...
+Rank [15/16]: Start tile-wise ring communication...
+Rank [9/16]: Start tile-wise ring communication...
+Rank [14/16]: Start tile-wise ring communication...
+Rank [15/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.6100444836774841 s
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Run 10
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks in global distance matrix...
+Rank [13/16]: Calculate diagonal blocks in global distance matrix...
+Rank [7/16]: Calculate diagonal blocks in global distance matrix...
+Rank [10/16]: Calculate diagonal blocks in global distance matrix...
+Rank [2/16]: Calculate diagonal blocks in global distance matrix...
+Rank [11/16]: Calculate diagonal blocks in global distance matrix...
+Rank [6/16]: Calculate diagonal blocks in global distance matrix...
+Rank [12/16]: Calculate diagonal blocks in global distance matrix...
+Rank [9/16]: Calculate diagonal blocks in global distance matrix...
+Rank [3/16]: Calculate diagonal blocks in global distance matrix...
+Rank [5/16]: Calculate diagonal blocks in global distance matrix...
+Rank [14/16]: Calculate diagonal blocks in global distance matrix...
+Rank [4/16]: Calculate diagonal blocks in global distance matrix...
+Rank [1/16]: Calculate diagonal blocks in global distance matrix...
+Rank [8/16]: Calculate diagonal blocks in global distance matrix...
+Rank [15/16]: Calculate diagonal blocks in global distance matrix...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [1/16]: Start tile-wise ring communication...
+Rank [2/16]: Start tile-wise ring communication...
+Rank [14/16]: Start tile-wise ring communication...
+Rank [8/16]: Start tile-wise ring communication...
+Rank [13/16]: Start tile-wise ring communication...
+Rank [4/16]: Start tile-wise ring communication...
+Rank [11/16]: Start tile-wise ring communication...
+Rank [3/16]: Start tile-wise ring communication...
+Rank [9/16]: Start tile-wise ring communication...
+Rank [10/16]: Start tile-wise ring communication...
+Rank [15/16]: Start tile-wise ring communication...
+Rank [6/16]: Start tile-wise ring communication...
+Rank [12/16]: Start tile-wise ring communication...
+Rank [5/16]: Start tile-wise ring communication...
+Rank [7/16]: Start tile-wise ring communication...
+Rank [15/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.6050152188399807 s
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Run 11
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks in global distance matrix...
+Rank [11/16]: Calculate diagonal blocks in global distance matrix...
+Rank [13/16]: Calculate diagonal blocks in global distance matrix...
+Rank [10/16]: Calculate diagonal blocks in global distance matrix...
+Rank [2/16]: Calculate diagonal blocks in global distance matrix...
+Rank [3/16]: Calculate diagonal blocks in global distance matrix...
+Rank [7/16]: Calculate diagonal blocks in global distance matrix...
+Rank [15/16]: Calculate diagonal blocks in global distance matrix...
+Rank [9/16]: Calculate diagonal blocks in global distance matrix...
+Rank [5/16]: Calculate diagonal blocks in global distance matrix...
+Rank [14/16]: Calculate diagonal blocks in global distance matrix...
+Rank [6/16]: Calculate diagonal blocks in global distance matrix...
+Rank [4/16]: Calculate diagonal blocks in global distance matrix...
+Rank [12/16]: Calculate diagonal blocks in global distance matrix...
+Rank [1/16]: Calculate diagonal blocks in global distance matrix...
+Rank [8/16]: Calculate diagonal blocks in global distance matrix...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [9/16]: Start tile-wise ring communication...
+Rank [5/16]: Start tile-wise ring communication...
+Rank [2/16]: Start tile-wise ring communication...
+Rank [4/16]: Start tile-wise ring communication...
+Rank [13/16]: Start tile-wise ring communication...
+Rank [8/16]: Start tile-wise ring communication...
+Rank [10/16]: Start tile-wise ring communication...
+Rank [7/16]: Start tile-wise ring communication...
+Rank [6/16]: Start tile-wise ring communication...
+Rank [11/16]: Start tile-wise ring communication...
+Rank [12/16]: Start tile-wise ring communication...
+Rank [3/16]: Start tile-wise ring communication...
+Rank [15/16]: Start tile-wise ring communication...
+Rank [14/16]: Start tile-wise ring communication...
+Rank [1/16]: Start tile-wise ring communication...
+Rank [15/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.6238499329192564 s
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+
+============================= JOB FEEDBACK =============================
+
+NodeName=uc2n[171,175,178-191]
+Job ID: 24674019
+Cluster: uc2
+User/Group: ku4408/scc
+State: COMPLETED (exit code 0)
+Nodes: 16
+Cores per node: 80
+CPU Utilized: 00:08:17
+CPU Efficiency: 0.40% of 1-10:50:40 core-walltime
+Job Wall-clock time: 00:01:38
+Memory Utilized: 1.10 GB
+Memory Efficiency: 0.08% of 1.37 TB
diff --git a/2_psrs_cdist/solutions/A1/strong_scaling/nodes_2.out b/2_psrs_cdist/solutions/A1/strong_scaling/nodes_2.out
new file mode 100644
index 0000000..cc5e465
--- /dev/null
+++ b/2_psrs_cdist/solutions/A1/strong_scaling/nodes_2.out
@@ -0,0 +1,257 @@
+Run 1
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [1/2]: Local data chunk has shape [25000, 18]...
+ [OK]
+Rank [0/2]: Local data chunk has shape [25000, 18]...
+Start distance calculations...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/2]: Calculate diagonal blocks in global distance matrix...
+Rank [1/2]: Calculate diagonal blocks in global distance matrix...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [25000, 50000].
+Process-averaged run time: 4.994515418075025 s
+Rank [1/2]: Local distance matrix has shape [25000, 50000].
+Run 2
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/2]: Local data chunk has shape [25000, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/2]: Local data chunk has shape [25000, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/2]: Calculate diagonal blocks in global distance matrix...
+Rank [1/2]: Calculate diagonal blocks in global distance matrix...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [25000, 50000].
+Process-averaged run time: 4.163690152112395 s
+Rank [1/2]: Local distance matrix has shape [25000, 50000].
+Run 3
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/2]: Local data chunk has shape [25000, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/2]: Local data chunk has shape [25000, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/2]: Calculate diagonal blocks in global distance matrix...
+Rank [1/2]: Calculate diagonal blocks in global distance matrix...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [25000, 50000].
+Process-averaged run time: 4.160190102178603 s
+Rank [1/2]: Local distance matrix has shape [25000, 50000].
+Run 4
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/2]: Local data chunk has shape [25000, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/2]: Local data chunk has shape [25000, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/2]: Calculate diagonal blocks in global distance matrix...
+Rank [1/2]: Calculate diagonal blocks in global distance matrix...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [25000, 50000].
+Process-averaged run time: 4.163418059237301 s
+Rank [1/2]: Local distance matrix has shape [25000, 50000].
+Run 5
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/2]: Local data chunk has shape [25000, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/2]: Local data chunk has shape [25000, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/2]: Calculate diagonal blocks in global distance matrix...
+Rank [1/2]: Calculate diagonal blocks in global distance matrix...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [25000, 50000].
+Process-averaged run time: 4.159190197940916 s
+Rank [1/2]: Local distance matrix has shape [25000, 50000].
+Run 6
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [1/2]: Local data chunk has shape [25000, 18]...
+ [OK]
+Rank [0/2]: Local data chunk has shape [25000, 18]...
+Start distance calculations...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/2]: Calculate diagonal blocks in global distance matrix...
+Rank [1/2]: Calculate diagonal blocks in global distance matrix...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [25000, 50000].
+Process-averaged run time: 4.163499476853758 s
+Rank [1/2]: Local distance matrix has shape [25000, 50000].
+Run 7
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/2]: Local data chunk has shape [25000, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/2]: Local data chunk has shape [25000, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/2]: Calculate diagonal blocks in global distance matrix...
+Rank [1/2]: Calculate diagonal blocks in global distance matrix...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [25000, 50000].
+Process-averaged run time: 4.1650856817141175 s
+Rank [1/2]: Local distance matrix has shape [25000, 50000].
+Run 8
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/2]: Local data chunk has shape [25000, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/2]: Local data chunk has shape [25000, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/2]: Calculate diagonal blocks in global distance matrix...
+Rank [1/2]: Calculate diagonal blocks in global distance matrix...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [25000, 50000].
+Process-averaged run time: 4.165958676021546 s
+Rank [1/2]: Local distance matrix has shape [25000, 50000].
+Run 9
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/2]: Local data chunk has shape [25000, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/2]: Local data chunk has shape [25000, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/2]: Calculate diagonal blocks in global distance matrix...
+Rank [1/2]: Calculate diagonal blocks in global distance matrix...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [25000, 50000].
+Process-averaged run time: 4.162757798098028 s
+Rank [1/2]: Local distance matrix has shape [25000, 50000].
+Run 10
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [1/2]: Local data chunk has shape [25000, 18]...
+ [OK]
+Rank [0/2]: Local data chunk has shape [25000, 18]...
+Start distance calculations...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/2]: Calculate diagonal blocks in global distance matrix...
+Rank [1/2]: Calculate diagonal blocks in global distance matrix...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [25000, 50000].
+Process-averaged run time: 4.164449342060834 s
+Rank [1/2]: Local distance matrix has shape [25000, 50000].
+Run 11
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/2]: Local data chunk has shape [25000, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/2]: Local data chunk has shape [25000, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/2]: Calculate diagonal blocks in global distance matrix...
+Rank [1/2]: Calculate diagonal blocks in global distance matrix...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [25000, 50000].
+Process-averaged run time: 4.169782547280192 s
+Rank [1/2]: Local distance matrix has shape [25000, 50000].
+
+============================= JOB FEEDBACK =============================
+
+NodeName=uc2n[259-260]
+Job ID: 24674015
+Cluster: uc2
+User/Group: ku4408/scc
+State: COMPLETED (exit code 0)
+Nodes: 2
+Cores per node: 80
+CPU Utilized: 00:02:21
+CPU Efficiency: 0.66% of 05:57:20 core-walltime
+Job Wall-clock time: 00:02:14
+Memory Utilized: 9.81 GB
+Memory Efficiency: 5.58% of 175.78 GB
diff --git a/2_psrs_cdist/solutions/A1/strong_scaling/nodes_32.out b/2_psrs_cdist/solutions/A1/strong_scaling/nodes_32.out
new file mode 100644
index 0000000..3e5ae43
--- /dev/null
+++ b/2_psrs_cdist/solutions/A1/strong_scaling/nodes_32.out
@@ -0,0 +1,2567 @@
+Run 1
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 32.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/32]: Local data chunk has shape [1562, 18]...
+Start distance calculations...
+ [OK]
+Rank [27/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [28/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [2/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [21/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [11/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [16/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [5/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [8/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [31/32]: Local data chunk has shape [1578, 18]...
+ [OK]
+Rank [30/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [26/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [23/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [7/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [6/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [15/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [18/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [13/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [29/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [14/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [22/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [4/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [17/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [3/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [10/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [25/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [24/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [20/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [12/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [9/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [1/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [19/32]: Local data chunk has shape [1562, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/32]: Calculate diagonal blocks in global distance matrix...
+Rank [7/32]: Calculate diagonal blocks in global distance matrix...
+Rank [9/32]: Calculate diagonal blocks in global distance matrix...
+Rank [26/32]: Calculate diagonal blocks in global distance matrix...
+Rank [25/32]: Calculate diagonal blocks in global distance matrix...
+Rank [20/32]: Calculate diagonal blocks in global distance matrix...
+Rank [8/32]: Calculate diagonal blocks in global distance matrix...
+Rank [30/32]: Calculate diagonal blocks in global distance matrix...
+Rank [23/32]: Calculate diagonal blocks in global distance matrix...
+Rank [31/32]: Calculate diagonal blocks in global distance matrix...
+Rank [6/32]: Calculate diagonal blocks in global distance matrix...
+Rank [27/32]: Calculate diagonal blocks in global distance matrix...
+Rank [19/32]: Calculate diagonal blocks in global distance matrix...
+Rank [15/32]: Calculate diagonal blocks in global distance matrix...
+Rank [2/32]: Calculate diagonal blocks in global distance matrix...
+Rank [18/32]: Calculate diagonal blocks in global distance matrix...
+Rank [13/32]: Calculate diagonal blocks in global distance matrix...
+Rank [14/32]: Calculate diagonal blocks in global distance matrix...
+Rank [5/32]: Calculate diagonal blocks in global distance matrix...
+Rank [1/32]: Calculate diagonal blocks in global distance matrix...
+Rank [28/32]: Calculate diagonal blocks in global distance matrix...
+Rank [12/32]: Calculate diagonal blocks in global distance matrix...
+Rank [21/32]: Calculate diagonal blocks in global distance matrix...
+Rank [29/32]: Calculate diagonal blocks in global distance matrix...
+Rank [11/32]: Calculate diagonal blocks in global distance matrix...
+Rank [22/32]: Calculate diagonal blocks in global distance matrix...
+Rank [16/32]: Calculate diagonal blocks in global distance matrix...
+Rank [4/32]: Calculate diagonal blocks in global distance matrix...
+Rank [17/32]: Calculate diagonal blocks in global distance matrix...
+Rank [3/32]: Calculate diagonal blocks in global distance matrix...
+Rank [10/32]: Calculate diagonal blocks in global distance matrix...
+Rank [24/32]: Calculate diagonal blocks in global distance matrix...
+Rank [8/32]: Start tile-wise ring communication...
+Rank [0/32]: Start tile-wise ring communication...
+Rank [24/32]: Start tile-wise ring communication...
+Rank [12/32]: Start tile-wise ring communication...
+Rank [16/32]: Start tile-wise ring communication...
+Rank [15/32]: Start tile-wise ring communication...
+Rank [13/32]: Start tile-wise ring communication...
+Rank [22/32]: Start tile-wise ring communication...
+Rank [17/32]: Start tile-wise ring communication...
+Rank [20/32]: Start tile-wise ring communication...
+Rank [18/32]: Start tile-wise ring communication...
+Rank [14/32]: Start tile-wise ring communication...
+Rank [26/32]: Start tile-wise ring communication...
+Rank [9/32]: Start tile-wise ring communication...
+Rank [25/32]: Start tile-wise ring communication...
+Rank [7/32]: Start tile-wise ring communication...
+Rank [5/32]: Start tile-wise ring communication...
+Rank [2/32]: Start tile-wise ring communication...
+Rank [30/32]: Start tile-wise ring communication...
+Rank [6/32]: Start tile-wise ring communication...
+Rank [21/32]: Start tile-wise ring communication...
+Rank [3/32]: Start tile-wise ring communication...
+Rank [10/32]: Start tile-wise ring communication...
+Rank [19/32]: Start tile-wise ring communication...
+Rank [27/32]: Start tile-wise ring communication...
+Rank [1/32]: Start tile-wise ring communication...
+Rank [4/32]: Start tile-wise ring communication...
+Rank [28/32]: Start tile-wise ring communication...
+Rank [31/32]: Start tile-wise ring communication...
+Rank [11/32]: Start tile-wise ring communication...
+Rank [23/32]: Start tile-wise ring communication...
+Rank [29/32]: Start tile-wise ring communication...
+Rank [18/32]: [DONE]
+Rank [15/32]: [DONE]
+Rank [7/32]: [DONE]
+Rank [14/32]: [DONE]
+Rank [13/32]: [DONE]
+Rank [2/32]: [DONE]
+Rank [19/32]: [DONE]
+Rank [6/32]: [DONE]
+Rank [12/32]: [DONE]
+Rank [17/32]: [DONE]
+Rank [20/32]: [DONE]
+Rank [16/32]: [DONE]
+Rank [8/32]: [DONE]
+Rank [11/32]: [DONE]
+Rank [10/32]: [DONE]
+Rank [23/32]: [DONE]
+Rank [21/32]: [DONE]
+Rank [3/32]: [DONE]
+Rank [22/32]: [DONE]
+Rank [31/32]: [DONE]
+Rank [5/32]: [DONE]
+Rank [0/32]: [DONE]
+Rank [4/32]: [DONE]
+Rank [9/32]: [DONE]
+Rank [1/32]: [DONE]
+Rank [24/32]: [DONE]
+Rank [26/32]: [DONE]
+Rank [28/32]: [DONE]
+Rank [25/32]: [DONE]
+Rank [30/32]: [DONE]
+Rank [27/32]: [DONE]
+Rank [29/32]: [DONE]
+Rank [0/32]: Local distance matrix has shape [1562, 50000].
+Process-averaged run time: 3.717215001495788 s
+Rank [24/32]: Local distance matrix has shape [1562, 50000].
+Rank [20/32]: Local distance matrix has shape [1562, 50000].
+Rank [8/32]: Local distance matrix has shape [1562, 50000].
+Rank [26/32]: Local distance matrix has shape [1562, 50000].
+Rank [6/32]: Local distance matrix has shape [1562, 50000].
+Rank [19/32]: Local distance matrix has shape [1562, 50000].
+Rank [28/32]: Local distance matrix has shape [1562, 50000].
+Rank [2/32]: Local distance matrix has shape [1562, 50000].
+Rank [12/32]: Local distance matrix has shape [1562, 50000].
+Rank [18/32]: Local distance matrix has shape [1562, 50000].
+Rank [14/32]: Local distance matrix has shape [1562, 50000].
+Rank [22/32]: Local distance matrix has shape [1562, 50000].
+Rank [16/32]: Local distance matrix has shape [1562, 50000].
+Rank [4/32]: Local distance matrix has shape [1562, 50000].
+Rank [17/32]: Local distance matrix has shape [1562, 50000].
+Rank [3/32]: Local distance matrix has shape [1562, 50000].
+Rank [10/32]: Local distance matrix has shape [1562, 50000].
+Rank [30/32]: Local distance matrix has shape [1562, 50000].
+Rank [23/32]: Local distance matrix has shape [1562, 50000].
+Rank [7/32]: Local distance matrix has shape [1562, 50000].
+Rank [31/32]: Local distance matrix has shape [1578, 50000].
+Rank [1/32]: Local distance matrix has shape [1562, 50000].
+Rank [27/32]: Local distance matrix has shape [1562, 50000].
+Rank [15/32]: Local distance matrix has shape [1562, 50000].
+Rank [21/32]: Local distance matrix has shape [1562, 50000].
+Rank [13/32]: Local distance matrix has shape [1562, 50000].
+Rank [29/32]: Local distance matrix has shape [1562, 50000].
+Rank [11/32]: Local distance matrix has shape [1562, 50000].
+Rank [5/32]: Local distance matrix has shape [1562, 50000].
+Rank [9/32]: Local distance matrix has shape [1562, 50000].
+Rank [25/32]: Local distance matrix has shape [1562, 50000].
+Run 2
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 32.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/32]: Local data chunk has shape [1562, 18]...
+Start distance calculations...
+ [OK]
+Rank [9/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [22/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [18/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [19/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [4/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [25/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [8/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [10/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [7/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [24/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [5/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [13/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [28/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [12/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [21/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [29/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [23/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [16/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [15/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [17/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [2/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [6/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [14/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [1/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [11/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [27/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [31/32]: Local data chunk has shape [1578, 18]...
+ [OK]
+Rank [20/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [26/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [30/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [3/32]: Local data chunk has shape [1562, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/32]: Calculate diagonal blocks in global distance matrix...
+Rank [28/32]: Calculate diagonal blocks in global distance matrix...
+Rank [7/32]: Calculate diagonal blocks in global distance matrix...
+Rank [6/32]: Calculate diagonal blocks in global distance matrix...
+Rank [5/32]: Calculate diagonal blocks in global distance matrix...
+Rank [3/32]: Calculate diagonal blocks in global distance matrix...
+Rank [13/32]: Calculate diagonal blocks in global distance matrix...
+Rank [16/32]: Calculate diagonal blocks in global distance matrix...
+Rank [14/32]: Calculate diagonal blocks in global distance matrix...
+Rank [15/32]: Calculate diagonal blocks in global distance matrix...
+Rank [11/32]: Calculate diagonal blocks in global distance matrix...
+Rank [21/32]: Calculate diagonal blocks in global distance matrix...
+Rank [31/32]: Calculate diagonal blocks in global distance matrix...
+Rank [17/32]: Calculate diagonal blocks in global distance matrix...
+Rank [30/32]: Calculate diagonal blocks in global distance matrix...
+Rank [1/32]: Calculate diagonal blocks in global distance matrix...
+Rank [29/32]: Calculate diagonal blocks in global distance matrix...
+Rank [26/32]: Calculate diagonal blocks in global distance matrix...
+Rank [8/32]: Calculate diagonal blocks in global distance matrix...
+Rank [23/32]: Calculate diagonal blocks in global distance matrix...
+Rank [24/32]: Calculate diagonal blocks in global distance matrix...
+Rank [20/32]: Calculate diagonal blocks in global distance matrix...
+Rank [19/32]: Calculate diagonal blocks in global distance matrix...
+Rank [22/32]: Calculate diagonal blocks in global distance matrix...
+Rank [12/32]: Calculate diagonal blocks in global distance matrix...
+Rank [18/32]: Calculate diagonal blocks in global distance matrix...
+Rank [27/32]: Calculate diagonal blocks in global distance matrix...
+Rank [4/32]: Calculate diagonal blocks in global distance matrix...
+Rank [9/32]: Calculate diagonal blocks in global distance matrix...
+Rank [25/32]: Calculate diagonal blocks in global distance matrix...
+Rank [2/32]: Calculate diagonal blocks in global distance matrix...
+Rank [10/32]: Calculate diagonal blocks in global distance matrix...
+Rank [0/32]: Start tile-wise ring communication...
+Rank [8/32]: Start tile-wise ring communication...
+Rank [16/32]: Start tile-wise ring communication...
+Rank [24/32]: Start tile-wise ring communication...
+Rank [28/32]: Start tile-wise ring communication...
+Rank [18/32]: Start tile-wise ring communication...
+Rank [31/32]: Start tile-wise ring communication...
+Rank [17/32]: Start tile-wise ring communication...
+Rank [30/32]: Start tile-wise ring communication...
+Rank [5/32]: Start tile-wise ring communication...
+Rank [4/32]: Start tile-wise ring communication...
+Rank [22/32]: Start tile-wise ring communication...
+Rank [27/32]: Start tile-wise ring communication...
+Rank [29/32]: Start tile-wise ring communication...
+Rank [26/32]: Start tile-wise ring communication...
+Rank [23/32]: Start tile-wise ring communication...
+Rank [20/32]: Start tile-wise ring communication...
+Rank [12/32]: Start tile-wise ring communication...
+Rank [9/32]: Start tile-wise ring communication...
+Rank [25/32]: Start tile-wise ring communication...
+Rank [2/32]: Start tile-wise ring communication...
+Rank [10/32]: Start tile-wise ring communication...
+Rank [6/32]: Start tile-wise ring communication...
+Rank [3/32]: Start tile-wise ring communication...
+Rank [14/32]: Start tile-wise ring communication...
+Rank [21/32]: Start tile-wise ring communication...
+Rank [1/32]: Start tile-wise ring communication...
+Rank [7/32]: Start tile-wise ring communication...
+Rank [15/32]: Start tile-wise ring communication...
+Rank [11/32]: Start tile-wise ring communication...
+Rank [19/32]: Start tile-wise ring communication...
+Rank [13/32]: Start tile-wise ring communication...
+Rank [2/32]: [DONE]
+Rank [7/32]: [DONE]
+Rank [15/32]: [DONE]
+Rank [23/32]: [DONE]
+Rank [12/32]: [DONE]
+Rank [6/32]: [DONE]
+Rank [3/32]: [DONE]
+Rank [8/32]: [DONE]
+Rank [5/32]: [DONE]
+Rank [14/32]: [DONE]
+Rank [4/32]: [DONE]
+Rank [0/32]: [DONE]
+Rank [31/32]: [DONE]
+Rank [13/32]: [DONE]
+Rank [17/32]: [DONE]
+Rank [18/32]: [DONE]
+Rank [11/32]: [DONE]
+Rank [1/32]: [DONE]
+Rank [9/32]: [DONE]
+Rank [20/32]: [DONE]
+Rank [19/32]: [DONE]
+Rank [21/32]: [DONE]
+Rank [16/32]: [DONE]
+Rank [22/32]: [DONE]
+Rank [10/32]: [DONE]
+Rank [28/32]: [DONE]
+Rank [26/32]: [DONE]
+Rank [24/32]: [DONE]
+Rank [29/32]: [DONE]
+Rank [25/32]: [DONE]
+Rank [27/32]: [DONE]
+Rank [0/32]: Local distance matrix has shape [1562, 50000].
+Process-averaged run time: 0.3523771684267558 s
+Rank [30/32]: [DONE]
+Rank [30/32]: Local distance matrix has shape [1562, 50000].
+Rank [2/32]: Local distance matrix has shape [1562, 50000].
+Rank [10/32]: Local distance matrix has shape [1562, 50000].
+Rank [6/32]: Local distance matrix has shape [1562, 50000].
+Rank [3/32]: Local distance matrix has shape [1562, 50000].
+Rank [16/32]: Local distance matrix has shape [1562, 50000].
+Rank [14/32]: Local distance matrix has shape [1562, 50000].
+Rank [7/32]: Local distance matrix has shape [1562, 50000].
+Rank [11/32]: Local distance matrix has shape [1562, 50000].
+Rank [21/32]: Local distance matrix has shape [1562, 50000].
+Rank [31/32]: Local distance matrix has shape [1578, 50000].
+Rank [28/32]: Local distance matrix has shape [1562, 50000].
+Rank [1/32]: Local distance matrix has shape [1562, 50000].
+Rank [29/32]: Local distance matrix has shape [1562, 50000].
+Rank [26/32]: Local distance matrix has shape [1562, 50000].
+Rank [8/32]: Local distance matrix has shape [1562, 50000].
+Rank [23/32]: Local distance matrix has shape [1562, 50000].
+Rank [24/32]: Local distance matrix has shape [1562, 50000].
+Rank [20/32]: Local distance matrix has shape [1562, 50000].
+Rank [19/32]: Local distance matrix has shape [1562, 50000].
+Rank [22/32]: Local distance matrix has shape [1562, 50000].
+Rank [18/32]: Local distance matrix has shape [1562, 50000].
+Rank [27/32]: Local distance matrix has shape [1562, 50000].
+Rank [17/32]: Local distance matrix has shape [1562, 50000].
+Rank [12/32]: Local distance matrix has shape [1562, 50000].
+Rank [4/32]: Local distance matrix has shape [1562, 50000].
+Rank [9/32]: Local distance matrix has shape [1562, 50000].
+Rank [25/32]: Local distance matrix has shape [1562, 50000].
+Rank [5/32]: Local distance matrix has shape [1562, 50000].
+Rank [13/32]: Local distance matrix has shape [1562, 50000].
+Rank [15/32]: Local distance matrix has shape [1562, 50000].
+Run 3
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 32.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/32]: Local data chunk has shape [1562, 18]...
+Start distance calculations...
+ [OK]
+Rank [23/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [28/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [8/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [18/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [9/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [22/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [20/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [26/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [16/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [17/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [6/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [14/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [24/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [3/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [21/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [25/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [30/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [19/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [27/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [10/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [11/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [2/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [31/32]: Local data chunk has shape [1578, 18]...
+ [OK]
+Rank [5/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [1/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [15/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [29/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [13/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [7/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [4/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [12/32]: Local data chunk has shape [1562, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/32]: Calculate diagonal blocks in global distance matrix...
+Rank [14/32]: Calculate diagonal blocks in global distance matrix...
+Rank [21/32]: Calculate diagonal blocks in global distance matrix...
+Rank [20/32]: Calculate diagonal blocks in global distance matrix...
+Rank [1/32]: Calculate diagonal blocks in global distance matrix...
+Rank [13/32]: Calculate diagonal blocks in global distance matrix...
+Rank [25/32]: Calculate diagonal blocks in global distance matrix...
+Rank [19/32]: Calculate diagonal blocks in global distance matrix...
+Rank [9/32]: Calculate diagonal blocks in global distance matrix...
+Rank [31/32]: Calculate diagonal blocks in global distance matrix...
+Rank [15/32]: Calculate diagonal blocks in global distance matrix...
+Rank [22/32]: Calculate diagonal blocks in global distance matrix...
+Rank [23/32]: Calculate diagonal blocks in global distance matrix...
+Rank [28/32]: Calculate diagonal blocks in global distance matrix...
+Rank [29/32]: Calculate diagonal blocks in global distance matrix...
+Rank [26/32]: Calculate diagonal blocks in global distance matrix...
+Rank [7/32]: Calculate diagonal blocks in global distance matrix...
+Rank [16/32]: Calculate diagonal blocks in global distance matrix...
+Rank [12/32]: Calculate diagonal blocks in global distance matrix...
+Rank [17/32]: Calculate diagonal blocks in global distance matrix...
+Rank [6/32]: Calculate diagonal blocks in global distance matrix...
+Rank [5/32]: Calculate diagonal blocks in global distance matrix...
+Rank [4/32]: Calculate diagonal blocks in global distance matrix...
+Rank [11/32]: Calculate diagonal blocks in global distance matrix...
+Rank [30/32]: Calculate diagonal blocks in global distance matrix...
+Rank [2/32]: Calculate diagonal blocks in global distance matrix...
+Rank [24/32]: Calculate diagonal blocks in global distance matrix...
+Rank [10/32]: Calculate diagonal blocks in global distance matrix...
+Rank [8/32]: Calculate diagonal blocks in global distance matrix...
+Rank [3/32]: Calculate diagonal blocks in global distance matrix...
+Rank [18/32]: Calculate diagonal blocks in global distance matrix...
+Rank [27/32]: Calculate diagonal blocks in global distance matrix...
+Rank [16/32]: Start tile-wise ring communication...
+Rank [8/32]: Start tile-wise ring communication...
+Rank [0/32]: Start tile-wise ring communication...
+Rank [24/32]: Start tile-wise ring communication...
+Rank [20/32]: Start tile-wise ring communication...
+Rank [3/32]: Start tile-wise ring communication...
+Rank [15/32]: Start tile-wise ring communication...
+Rank [23/32]: Start tile-wise ring communication...
+Rank [17/32]: Start tile-wise ring communication...
+Rank [31/32]: Start tile-wise ring communication...
+Rank [10/32]: Start tile-wise ring communication...
+Rank [5/32]: Start tile-wise ring communication...
+Rank [2/32]: Start tile-wise ring communication...
+Rank [28/32]: Start tile-wise ring communication...
+Rank [29/32]: Start tile-wise ring communication...
+Rank [4/32]: Start tile-wise ring communication...
+Rank [18/32]: Start tile-wise ring communication...
+Rank [27/32]: Start tile-wise ring communication...
+Rank [1/32]: Start tile-wise ring communication...
+Rank [25/32]: Start tile-wise ring communication...
+Rank [22/32]: Start tile-wise ring communication...
+Rank [21/32]: Start tile-wise ring communication...
+Rank [26/32]: Start tile-wise ring communication...
+Rank [7/32]: Start tile-wise ring communication...
+Rank [19/32]: Start tile-wise ring communication...
+Rank [9/32]: Start tile-wise ring communication...
+Rank [12/32]: Start tile-wise ring communication...
+Rank [14/32]: Start tile-wise ring communication...
+Rank [11/32]: Start tile-wise ring communication...
+Rank [30/32]: Start tile-wise ring communication...
+Rank [13/32]: Start tile-wise ring communication...
+Rank [6/32]: Start tile-wise ring communication...
+Rank [12/32]: [DONE]
+Rank [2/32]: [DONE]
+Rank [7/32]: [DONE]
+Rank [13/32]: [DONE]
+Rank [14/32]: [DONE]
+Rank [9/32]: [DONE]
+Rank [11/32]: [DONE]
+Rank [23/32]: [DONE]
+Rank [6/32]: [DONE]
+Rank [31/32]: [DONE]
+Rank [10/32]: [DONE]
+Rank [3/32]: [DONE]
+Rank [8/32]: [DONE]
+Rank [5/32]: [DONE]
+Rank [22/32]: [DONE]
+Rank [4/32]: [DONE]
+Rank [21/32]: [DONE]
+Rank [1/32]: [DONE]
+Rank [15/32]: [DONE]
+Rank [0/32]: [DONE]
+Rank [18/32]: [DONE]
+Rank [17/32]: [DONE]
+Rank [19/32]: [DONE]
+Rank [16/32]: [DONE]
+Rank [20/32]: [DONE]
+Rank [24/32]: [DONE]
+Rank [25/32]: [DONE]
+Rank [26/32]: [DONE]
+Rank [28/32]: [DONE]
+Rank [29/32]: [DONE]
+Rank [27/32]: [DONE]
+Rank [30/32]: [DONE]
+Rank [0/32]: Local distance matrix has shape [1562, 50000].
+Process-averaged run time: 0.3502360037819017 s
+Rank [30/32]: Local distance matrix has shape [1562, 50000].
+Rank [14/32]: Local distance matrix has shape [1562, 50000].
+Rank [10/32]: Local distance matrix has shape [1562, 50000].
+Rank [21/32]: Local distance matrix has shape [1562, 50000].
+Rank [17/32]: Local distance matrix has shape [1562, 50000].
+Rank [6/32]: Local distance matrix has shape [1562, 50000].
+Rank [5/32]: Local distance matrix has shape [1562, 50000].
+Rank [4/32]: Local distance matrix has shape [1562, 50000].
+Rank [11/32]: Local distance matrix has shape [1562, 50000].
+Rank [2/32]: Local distance matrix has shape [1562, 50000].
+Rank [24/32]: Local distance matrix has shape [1562, 50000].
+Rank [8/32]: Local distance matrix has shape [1562, 50000].
+Rank [3/32]: Local distance matrix has shape [1562, 50000].
+Rank [18/32]: Local distance matrix has shape [1562, 50000].
+Rank [27/32]: Local distance matrix has shape [1562, 50000].
+Rank [1/32]: Local distance matrix has shape [1562, 50000].
+Rank [13/32]: Local distance matrix has shape [1562, 50000].
+Rank [25/32]: Local distance matrix has shape [1562, 50000].
+Rank [19/32]: Local distance matrix has shape [1562, 50000].
+Rank [9/32]: Local distance matrix has shape [1562, 50000].
+Rank [31/32]: Local distance matrix has shape [1578, 50000].
+Rank [15/32]: Local distance matrix has shape [1562, 50000].
+Rank [23/32]: Local distance matrix has shape [1562, 50000].
+Rank [28/32]: Local distance matrix has shape [1562, 50000].
+Rank [29/32]: Local distance matrix has shape [1562, 50000].
+Rank [20/32]: Local distance matrix has shape [1562, 50000].
+Rank [26/32]: Local distance matrix has shape [1562, 50000].
+Rank [7/32]: Local distance matrix has shape [1562, 50000].
+Rank [16/32]: Local distance matrix has shape [1562, 50000].
+Rank [12/32]: Local distance matrix has shape [1562, 50000].
+Rank [22/32]: Local distance matrix has shape [1562, 50000].
+Run 4
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 32.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [7/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [0/32]: Local data chunk has shape [1562, 18]...
+Start distance calculations...
+ [OK]
+Rank [17/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [18/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [8/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [4/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [22/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [1/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [6/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [21/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [3/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [16/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [5/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [29/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [30/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [12/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [27/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [26/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [14/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [24/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [19/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [25/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [2/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [13/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [23/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [9/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [11/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [20/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [10/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [31/32]: Local data chunk has shape [1578, 18]...
+ [OK]
+Rank [28/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [15/32]: Local data chunk has shape [1562, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/32]: Calculate diagonal blocks in global distance matrix...
+Rank [2/32]: Calculate diagonal blocks in global distance matrix...
+Rank [15/32]: Calculate diagonal blocks in global distance matrix...
+Rank [22/32]: Calculate diagonal blocks in global distance matrix...
+Rank [23/32]: Calculate diagonal blocks in global distance matrix...
+Rank [18/32]: Calculate diagonal blocks in global distance matrix...
+Rank [21/32]: Calculate diagonal blocks in global distance matrix...
+Rank [25/32]: Calculate diagonal blocks in global distance matrix...
+Rank [7/32]: Calculate diagonal blocks in global distance matrix...
+Rank [3/32]: Calculate diagonal blocks in global distance matrix...
+Rank [31/32]: Calculate diagonal blocks in global distance matrix...
+Rank [17/32]: Calculate diagonal blocks in global distance matrix...
+Rank [26/32]: Calculate diagonal blocks in global distance matrix...
+Rank [28/32]: Calculate diagonal blocks in global distance matrix...
+Rank [14/32]: Calculate diagonal blocks in global distance matrix...
+Rank [29/32]: Calculate diagonal blocks in global distance matrix...
+Rank [1/32]: Calculate diagonal blocks in global distance matrix...
+Rank [20/32]: Calculate diagonal blocks in global distance matrix...
+Rank [24/32]: Calculate diagonal blocks in global distance matrix...
+Rank [8/32]: Calculate diagonal blocks in global distance matrix...
+Rank [19/32]: Calculate diagonal blocks in global distance matrix...
+Rank [30/32]: Calculate diagonal blocks in global distance matrix...
+Rank [12/32]: Calculate diagonal blocks in global distance matrix...
+Rank [10/32]: Calculate diagonal blocks in global distance matrix...
+Rank [6/32]: Calculate diagonal blocks in global distance matrix...
+Rank [5/32]: Calculate diagonal blocks in global distance matrix...
+Rank [4/32]: Calculate diagonal blocks in global distance matrix...
+Rank [9/32]: Calculate diagonal blocks in global distance matrix...
+Rank [16/32]: Calculate diagonal blocks in global distance matrix...
+Rank [11/32]: Calculate diagonal blocks in global distance matrix...
+Rank [13/32]: Calculate diagonal blocks in global distance matrix...
+Rank [27/32]: Calculate diagonal blocks in global distance matrix...
+Rank [0/32]: Start tile-wise ring communication...
+Rank [16/32]: Start tile-wise ring communication...
+Rank [24/32]: Start tile-wise ring communication...
+Rank [8/32]: Start tile-wise ring communication...
+Rank [28/32]: Start tile-wise ring communication...
+Rank [20/32]: Start tile-wise ring communication...
+Rank [11/32]: Start tile-wise ring communication...
+Rank [15/32]: Start tile-wise ring communication...
+Rank [4/32]: Start tile-wise ring communication...
+Rank [12/32]: Start tile-wise ring communication...
+Rank [25/32]: Start tile-wise ring communication...
+Rank [27/32]: Start tile-wise ring communication...
+Rank [22/32]: Start tile-wise ring communication...
+Rank [6/32]: Start tile-wise ring communication...
+Rank [18/32]: Start tile-wise ring communication...
+Rank [21/32]: Start tile-wise ring communication...
+Rank [1/32]: Start tile-wise ring communication...
+Rank [17/32]: Start tile-wise ring communication...
+Rank [26/32]: Start tile-wise ring communication...
+Rank [7/32]: Start tile-wise ring communication...
+Rank [10/32]: Start tile-wise ring communication...
+Rank [2/32]: Start tile-wise ring communication...
+Rank [30/32]: Start tile-wise ring communication...
+Rank [23/32]: Start tile-wise ring communication...
+Rank [19/32]: Start tile-wise ring communication...
+Rank [9/32]: Start tile-wise ring communication...
+Rank [3/32]: Start tile-wise ring communication...
+Rank [14/32]: Start tile-wise ring communication...
+Rank [29/32]: Start tile-wise ring communication...
+Rank [31/32]: Start tile-wise ring communication...
+Rank [5/32]: Start tile-wise ring communication...
+Rank [13/32]: Start tile-wise ring communication...
+Rank [2/32]: [DONE]
+Rank [7/32]: [DONE]
+Rank [0/32]: [DONE]
+Rank [31/32]: [DONE]
+Rank [23/32]: [DONE]
+Rank [14/32]: [DONE]
+Rank [5/32]: [DONE]
+Rank [3/32]: [DONE]
+Rank [15/32]: [DONE]
+Rank [12/32]: [DONE]
+Rank [4/32]: [DONE]
+Rank [11/32]: [DONE]
+Rank [9/32]: [DONE]
+Rank [6/32]: [DONE]
+Rank [1/32]: [DONE]
+Rank [8/32]: [DONE]
+Rank [13/32]: [DONE]
+Rank [21/32]: [DONE]
+Rank [10/32]: [DONE]
+Rank [22/32]: [DONE]
+Rank [17/32]: [DONE]
+Rank [16/32]: [DONE]
+Rank [18/32]: [DONE]
+Rank [20/32]: [DONE]
+Rank [19/32]: [DONE]
+Rank [26/32]: [DONE]
+Rank [28/32]: [DONE]
+Rank [24/32]: [DONE]
+Rank [29/32]: [DONE]
+Rank [25/32]: [DONE]
+Rank [27/32]: [DONE]
+Rank [30/32]: [DONE]
+Rank [0/32]: Local distance matrix has shape [1562, 50000].
+Process-averaged run time: 0.3601568423619028 s
+Rank [30/32]: Local distance matrix has shape [1562, 50000].
+Rank [20/32]: Local distance matrix has shape [1562, 50000].
+Rank [26/32]: Local distance matrix has shape [1562, 50000].
+Rank [17/32]: Local distance matrix has shape [1562, 50000].
+Rank [14/32]: Local distance matrix has shape [1562, 50000].
+Rank [12/32]: Local distance matrix has shape [1562, 50000].
+Rank [25/32]: Local distance matrix has shape [1562, 50000].
+Rank [10/32]: Local distance matrix has shape [1562, 50000].
+Rank [6/32]: Local distance matrix has shape [1562, 50000].
+Rank [5/32]: Local distance matrix has shape [1562, 50000].
+Rank [4/32]: Local distance matrix has shape [1562, 50000].
+Rank [9/32]: Local distance matrix has shape [1562, 50000].
+Rank [16/32]: Local distance matrix has shape [1562, 50000].
+Rank [13/32]: Local distance matrix has shape [1562, 50000].
+Rank [15/32]: Local distance matrix has shape [1562, 50000].
+Rank [24/32]: Local distance matrix has shape [1562, 50000].
+Rank [18/32]: Local distance matrix has shape [1562, 50000].
+Rank [8/32]: Local distance matrix has shape [1562, 50000].
+Rank [21/32]: Local distance matrix has shape [1562, 50000].
+Rank [1/32]: Local distance matrix has shape [1562, 50000].
+Rank [19/32]: Local distance matrix has shape [1562, 50000].
+Rank [7/32]: Local distance matrix has shape [1562, 50000].
+Rank [2/32]: Local distance matrix has shape [1562, 50000].
+Rank [11/32]: Local distance matrix has shape [1562, 50000].
+Rank [22/32]: Local distance matrix has shape [1562, 50000].
+Rank [31/32]: Local distance matrix has shape [1578, 50000].
+Rank [28/32]: Local distance matrix has shape [1562, 50000].
+Rank [3/32]: Local distance matrix has shape [1562, 50000].
+Rank [27/32]: Local distance matrix has shape [1562, 50000].
+Rank [29/32]: Local distance matrix has shape [1562, 50000].
+Rank [23/32]: Local distance matrix has shape [1562, 50000].
+Run 5
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 32.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/32]: Local data chunk has shape [1562, 18]...
+Start distance calculations...
+ [OK]
+Rank [5/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [22/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [8/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [18/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [4/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [1/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [29/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [7/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [9/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [3/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [30/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [19/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [20/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [27/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [23/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [14/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [28/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [17/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [25/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [24/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [16/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [6/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [26/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [10/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [2/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [12/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [31/32]: Local data chunk has shape [1578, 18]...
+ [OK]
+Rank [21/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [13/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [15/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [11/32]: Local data chunk has shape [1562, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/32]: Calculate diagonal blocks in global distance matrix...
+Rank [16/32]: Calculate diagonal blocks in global distance matrix...
+Rank [22/32]: Calculate diagonal blocks in global distance matrix...
+Rank [12/32]: Calculate diagonal blocks in global distance matrix...
+Rank [4/32]: Calculate diagonal blocks in global distance matrix...
+Rank [2/32]: Calculate diagonal blocks in global distance matrix...
+Rank [24/32]: Calculate diagonal blocks in global distance matrix...
+Rank [9/32]: Calculate diagonal blocks in global distance matrix...
+Rank [14/32]: Calculate diagonal blocks in global distance matrix...
+Rank [8/32]: Calculate diagonal blocks in global distance matrix...
+Rank [18/32]: Calculate diagonal blocks in global distance matrix...
+Rank [10/32]: Calculate diagonal blocks in global distance matrix...
+Rank [1/32]: Calculate diagonal blocks in global distance matrix...
+Rank [19/32]: Calculate diagonal blocks in global distance matrix...
+Rank [21/32]: Calculate diagonal blocks in global distance matrix...
+Rank [13/32]: Calculate diagonal blocks in global distance matrix...
+Rank [7/32]: Calculate diagonal blocks in global distance matrix...
+Rank [27/32]: Calculate diagonal blocks in global distance matrix...
+Rank [28/32]: Calculate diagonal blocks in global distance matrix...
+Rank [30/32]: Calculate diagonal blocks in global distance matrix...
+Rank [5/32]: Calculate diagonal blocks in global distance matrix...
+Rank [29/32]: Calculate diagonal blocks in global distance matrix...
+Rank [31/32]: Calculate diagonal blocks in global distance matrix...
+Rank [23/32]: Calculate diagonal blocks in global distance matrix...
+Rank [26/32]: Calculate diagonal blocks in global distance matrix...
+Rank [25/32]: Calculate diagonal blocks in global distance matrix...
+Rank [17/32]: Calculate diagonal blocks in global distance matrix...
+Rank [15/32]: Calculate diagonal blocks in global distance matrix...
+Rank [20/32]: Calculate diagonal blocks in global distance matrix...
+Rank [6/32]: Calculate diagonal blocks in global distance matrix...
+Rank [11/32]: Calculate diagonal blocks in global distance matrix...
+Rank [3/32]: Calculate diagonal blocks in global distance matrix...
+Rank [16/32]: Start tile-wise ring communication...
+Rank [0/32]: Start tile-wise ring communication...
+Rank [24/32]: Start tile-wise ring communication...
+Rank [8/32]: Start tile-wise ring communication...
+Rank [11/32]: Start tile-wise ring communication...
+Rank [2/32]: Start tile-wise ring communication...
+Rank [6/32]: Start tile-wise ring communication...
+Rank [5/32]: Start tile-wise ring communication...
+Rank [26/32]: Start tile-wise ring communication...
+Rank [30/32]: Start tile-wise ring communication...
+Rank [14/32]: Start tile-wise ring communication...
+Rank [31/32]: Start tile-wise ring communication...
+Rank [15/32]: Start tile-wise ring communication...
+Rank [20/32]: Start tile-wise ring communication...
+Rank [4/32]: Start tile-wise ring communication...
+Rank [28/32]: Start tile-wise ring communication...
+Rank [27/32]: Start tile-wise ring communication...
+Rank [18/32]: Start tile-wise ring communication...
+Rank [10/32]: Start tile-wise ring communication...
+Rank [1/32]: Start tile-wise ring communication...
+Rank [19/32]: Start tile-wise ring communication...
+Rank [25/32]: Start tile-wise ring communication...
+Rank [22/32]: Start tile-wise ring communication...
+Rank [3/32]: Start tile-wise ring communication...
+Rank [29/32]: Start tile-wise ring communication...
+Rank [23/32]: Start tile-wise ring communication...
+Rank [21/32]: Start tile-wise ring communication...
+Rank [13/32]: Start tile-wise ring communication...
+Rank [7/32]: Start tile-wise ring communication...
+Rank [12/32]: Start tile-wise ring communication...
+Rank [9/32]: Start tile-wise ring communication...
+Rank [17/32]: Start tile-wise ring communication...
+Rank [4/32]: [DONE]
+Rank [3/32]: [DONE]
+Rank [2/32]: [DONE]
+Rank [5/32]: [DONE]
+Rank [14/32]: [DONE]
+Rank [13/32]: [DONE]
+Rank [12/32]: [DONE]
+Rank [7/32]: [DONE]
+Rank [23/32]: [DONE]
+Rank [6/32]: [DONE]
+Rank [11/32]: [DONE]
+Rank [10/32]: [DONE]
+Rank [8/32]: [DONE]
+Rank [21/32]: [DONE]
+Rank [15/32]: [DONE]
+Rank [22/32]: [DONE]
+Rank [31/32]: [DONE]
+Rank [9/32]: [DONE]
+Rank [18/32]: [DONE]
+Rank [17/32]: [DONE]
+Rank [0/32]: [DONE]
+Rank [1/32]: [DONE]
+Rank [20/32]: [DONE]
+Rank [16/32]: [DONE]
+Rank [19/32]: [DONE]
+Rank [30/32]: [DONE]
+Rank [24/32]: [DONE]
+Rank [26/32]: [DONE]
+Rank [25/32]: [DONE]
+Rank [29/32]: [DONE]
+Rank [28/32]: [DONE]
+Rank [27/32]: [DONE]
+Rank [0/32]: Local distance matrix has shape [1562, 50000].
+Process-averaged run time: 0.3551012375683058 s
+Rank [28/32]: Local distance matrix has shape [1562, 50000].
+Rank [27/32]: Local distance matrix has shape [1562, 50000].
+Rank [30/32]: Local distance matrix has shape [1562, 50000].
+Rank [2/32]: Local distance matrix has shape [1562, 50000].
+Rank [24/32]: Local distance matrix has shape [1562, 50000].
+Rank [9/32]: Local distance matrix has shape [1562, 50000].
+Rank [8/32]: Local distance matrix has shape [1562, 50000].
+Rank [18/32]: Local distance matrix has shape [1562, 50000].
+Rank [10/32]: Local distance matrix has shape [1562, 50000].
+Rank [31/32]: Local distance matrix has shape [1578, 50000].
+Rank [1/32]: Local distance matrix has shape [1562, 50000].
+Rank [19/32]: Local distance matrix has shape [1562, 50000].
+Rank [21/32]: Local distance matrix has shape [1562, 50000].
+Rank [13/32]: Local distance matrix has shape [1562, 50000].
+Rank [7/32]: Local distance matrix has shape [1562, 50000].
+Rank [25/32]: Local distance matrix has shape [1562, 50000].
+Rank [17/32]: Local distance matrix has shape [1562, 50000].
+Rank [20/32]: Local distance matrix has shape [1562, 50000].
+Rank [16/32]: Local distance matrix has shape [1562, 50000].
+Rank [12/32]: Local distance matrix has shape [1562, 50000].
+Rank [4/32]: Local distance matrix has shape [1562, 50000].
+Rank [6/32]: Local distance matrix has shape [1562, 50000].
+Rank [3/32]: Local distance matrix has shape [1562, 50000].
+Rank [23/32]: Local distance matrix has shape [1562, 50000].
+Rank [26/32]: Local distance matrix has shape [1562, 50000].
+Rank [15/32]: Local distance matrix has shape [1562, 50000].
+Rank [22/32]: Local distance matrix has shape [1562, 50000].
+Rank [11/32]: Local distance matrix has shape [1562, 50000].
+Rank [5/32]: Local distance matrix has shape [1562, 50000].
+Rank [29/32]: Local distance matrix has shape [1562, 50000].
+Rank [14/32]: Local distance matrix has shape [1562, 50000].
+Run 6
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 32.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [3/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [0/32]: Local data chunk has shape [1562, 18]...
+Start distance calculations...
+ [OK]
+Rank [4/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [23/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [27/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [24/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [11/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [28/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [25/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [2/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [14/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [7/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [10/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [15/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [13/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [1/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [17/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [21/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [31/32]: Local data chunk has shape [1578, 18]...
+ [OK]
+Rank [6/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [20/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [19/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [18/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [9/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [16/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [30/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [29/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [22/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [5/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [26/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [12/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [8/32]: Local data chunk has shape [1562, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/32]: Calculate diagonal blocks in global distance matrix...
+Rank [22/32]: Calculate diagonal blocks in global distance matrix...
+Rank [20/32]: Calculate diagonal blocks in global distance matrix...
+Rank [14/32]: Calculate diagonal blocks in global distance matrix...
+Rank [16/32]: Calculate diagonal blocks in global distance matrix...
+Rank [24/32]: Calculate diagonal blocks in global distance matrix...
+Rank [28/32]: Calculate diagonal blocks in global distance matrix...
+Rank [13/32]: Calculate diagonal blocks in global distance matrix...
+Rank [17/32]: Calculate diagonal blocks in global distance matrix...
+Rank [23/32]: Calculate diagonal blocks in global distance matrix...
+Rank [19/32]: Calculate diagonal blocks in global distance matrix...
+Rank [2/32]: Calculate diagonal blocks in global distance matrix...
+Rank [9/32]: Calculate diagonal blocks in global distance matrix...
+Rank [31/32]: Calculate diagonal blocks in global distance matrix...
+Rank [6/32]: Calculate diagonal blocks in global distance matrix...
+Rank [8/32]: Calculate diagonal blocks in global distance matrix...
+Rank [5/32]: Calculate diagonal blocks in global distance matrix...
+Rank [7/32]: Calculate diagonal blocks in global distance matrix...
+Rank [10/32]: Calculate diagonal blocks in global distance matrix...
+Rank [27/32]: Calculate diagonal blocks in global distance matrix...
+Rank [11/32]: Calculate diagonal blocks in global distance matrix...
+Rank [3/32]: Calculate diagonal blocks in global distance matrix...
+Rank [15/32]: Calculate diagonal blocks in global distance matrix...
+Rank [25/32]: Calculate diagonal blocks in global distance matrix...
+Rank [4/32]: Calculate diagonal blocks in global distance matrix...
+Rank [1/32]: Calculate diagonal blocks in global distance matrix...
+Rank [29/32]: Calculate diagonal blocks in global distance matrix...
+Rank [12/32]: Calculate diagonal blocks in global distance matrix...
+Rank [26/32]: Calculate diagonal blocks in global distance matrix...
+Rank [21/32]: Calculate diagonal blocks in global distance matrix...
+Rank [18/32]: Calculate diagonal blocks in global distance matrix...
+Rank [30/32]: Calculate diagonal blocks in global distance matrix...
+Rank [16/32]: Start tile-wise ring communication...
+Rank [0/32]: Start tile-wise ring communication...
+Rank [8/32]: Start tile-wise ring communication...
+Rank [24/32]: Start tile-wise ring communication...
+Rank [13/32]: Start tile-wise ring communication...
+Rank [11/32]: Start tile-wise ring communication...
+Rank [31/32]: Start tile-wise ring communication...
+Rank [5/32]: Start tile-wise ring communication...
+Rank [15/32]: Start tile-wise ring communication...
+Rank [7/32]: Start tile-wise ring communication...
+Rank [22/32]: Start tile-wise ring communication...
+Rank [27/32]: Start tile-wise ring communication...
+Rank [23/32]: Start tile-wise ring communication...
+Rank [28/32]: Start tile-wise ring communication...
+Rank [18/32]: Start tile-wise ring communication...
+Rank [21/32]: Start tile-wise ring communication...
+Rank [3/32]: Start tile-wise ring communication...
+Rank [20/32]: Start tile-wise ring communication...
+Rank [17/32]: Start tile-wise ring communication...
+Rank [25/32]: Start tile-wise ring communication...
+Rank [9/32]: Start tile-wise ring communication...
+Rank [4/32]: Start tile-wise ring communication...
+Rank [14/32]: Start tile-wise ring communication...
+Rank [10/32]: Start tile-wise ring communication...
+Rank [2/32]: Start tile-wise ring communication...
+Rank [30/32]: Start tile-wise ring communication...
+Rank [29/32]: Start tile-wise ring communication...
+Rank [19/32]: Start tile-wise ring communication...
+Rank [26/32]: Start tile-wise ring communication...
+Rank [12/32]: Start tile-wise ring communication...
+Rank [6/32]: Start tile-wise ring communication...
+Rank [1/32]: Start tile-wise ring communication...
+Rank [15/32]: [DONE]
+Rank [16/32]: [DONE]
+Rank [17/32]: [DONE]
+Rank [18/32]: [DONE]
+Rank [2/32]: [DONE]
+Rank [23/32]: [DONE]
+Rank [14/32]: [DONE]
+Rank [19/32]: [DONE]
+Rank [12/32]: [DONE]
+Rank [20/32]: [DONE]
+Rank [13/32]: [DONE]
+Rank [11/32]: [DONE]
+Rank [3/32]: [DONE]
+Rank [9/32]: [DONE]
+Rank [7/32]: [DONE]
+Rank [5/32]: [DONE]
+Rank [10/32]: [DONE]
+Rank [21/32]: [DONE]
+Rank [31/32]: [DONE]
+Rank [6/32]: [DONE]
+Rank [0/32]: [DONE]
+Rank [1/32]: [DONE]
+Rank [8/32]: [DONE]
+Rank [4/32]: [DONE]
+Rank [22/32]: [DONE]
+Rank [25/32]: [DONE]
+Rank [26/32]: [DONE]
+Rank [24/32]: [DONE]
+Rank [28/32]: [DONE]
+Rank [29/32]: [DONE]
+Rank [30/32]: [DONE]
+Rank [27/32]: [DONE]
+Rank [0/32]: Local distance matrix has shape [1562, 50000].
+Process-averaged run time: 0.35944829144864343 s
+Rank [30/32]: Local distance matrix has shape [1562, 50000].
+Rank [27/32]: Local distance matrix has shape [1562, 50000].
+Rank [24/32]: Local distance matrix has shape [1562, 50000].
+Rank [28/32]: Local distance matrix has shape [1562, 50000].
+Rank [25/32]: Local distance matrix has shape [1562, 50000].
+Rank [1/32]: Local distance matrix has shape [1562, 50000].
+Rank [29/32]: Local distance matrix has shape [1562, 50000].
+Rank [26/32]: Local distance matrix has shape [1562, 50000].
+Rank [22/32]: Local distance matrix has shape [1562, 50000].
+Rank [31/32]: Local distance matrix has shape [1578, 50000].
+Rank [6/32]: Local distance matrix has shape [1562, 50000].
+Rank [8/32]: Local distance matrix has shape [1562, 50000].
+Rank [5/32]: Local distance matrix has shape [1562, 50000].
+Rank [14/32]: Local distance matrix has shape [1562, 50000].
+Rank [16/32]: Local distance matrix has shape [1562, 50000].
+Rank [7/32]: Local distance matrix has shape [1562, 50000].
+Rank [10/32]: Local distance matrix has shape [1562, 50000].
+Rank [11/32]: Local distance matrix has shape [1562, 50000].
+Rank [3/32]: Local distance matrix has shape [1562, 50000].
+Rank [15/32]: Local distance matrix has shape [1562, 50000].
+Rank [4/32]: Local distance matrix has shape [1562, 50000].
+Rank [13/32]: Local distance matrix has shape [1562, 50000].
+Rank [17/32]: Local distance matrix has shape [1562, 50000].
+Rank [20/32]: Local distance matrix has shape [1562, 50000].
+Rank [23/32]: Local distance matrix has shape [1562, 50000].
+Rank [19/32]: Local distance matrix has shape [1562, 50000].
+Rank [12/32]: Local distance matrix has shape [1562, 50000].
+Rank [21/32]: Local distance matrix has shape [1562, 50000].
+Rank [2/32]: Local distance matrix has shape [1562, 50000].
+Rank [18/32]: Local distance matrix has shape [1562, 50000].
+Rank [9/32]: Local distance matrix has shape [1562, 50000].
+Run 7
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 32.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [17/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [24/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [27/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [3/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [21/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [23/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [1/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [11/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [0/32]: Local data chunk has shape [1562, 18]...
+Start distance calculations...
+ [OK]
+Rank [14/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [20/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [31/32]: Local data chunk has shape [1578, 18]...
+ [OK]
+Rank [22/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [8/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [4/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [25/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [5/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [28/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [12/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [16/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [2/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [29/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [18/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [10/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [13/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [6/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [26/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [19/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [30/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [9/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [7/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [15/32]: Local data chunk has shape [1562, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/32]: Calculate diagonal blocks in global distance matrix...
+Rank [20/32]: Calculate diagonal blocks in global distance matrix...
+Rank [8/32]: Calculate diagonal blocks in global distance matrix...
+Rank [9/32]: Calculate diagonal blocks in global distance matrix...
+Rank [12/32]: Calculate diagonal blocks in global distance matrix...
+Rank [28/32]: Calculate diagonal blocks in global distance matrix...
+Rank [19/32]: Calculate diagonal blocks in global distance matrix...
+Rank [18/32]: Calculate diagonal blocks in global distance matrix...
+Rank [13/32]: Calculate diagonal blocks in global distance matrix...
+Rank [17/32]: Calculate diagonal blocks in global distance matrix...
+Rank [26/32]: Calculate diagonal blocks in global distance matrix...
+Rank [30/32]: Calculate diagonal blocks in global distance matrix...
+Rank [24/32]: Calculate diagonal blocks in global distance matrix...
+Rank [14/32]: Calculate diagonal blocks in global distance matrix...
+Rank [31/32]: Calculate diagonal blocks in global distance matrix...
+Rank [22/32]: Calculate diagonal blocks in global distance matrix...
+Rank [6/32]: Calculate diagonal blocks in global distance matrix...
+Rank [21/32]: Calculate diagonal blocks in global distance matrix...
+Rank [23/32]: Calculate diagonal blocks in global distance matrix...
+Rank [1/32]: Calculate diagonal blocks in global distance matrix...
+Rank [10/32]: Calculate diagonal blocks in global distance matrix...
+Rank [27/32]: Calculate diagonal blocks in global distance matrix...
+Rank [7/32]: Calculate diagonal blocks in global distance matrix...
+Rank [4/32]: Calculate diagonal blocks in global distance matrix...
+Rank [3/32]: Calculate diagonal blocks in global distance matrix...
+Rank [25/32]: Calculate diagonal blocks in global distance matrix...
+Rank [16/32]: Calculate diagonal blocks in global distance matrix...
+Rank [2/32]: Calculate diagonal blocks in global distance matrix...
+Rank [5/32]: Calculate diagonal blocks in global distance matrix...
+Rank [15/32]: Calculate diagonal blocks in global distance matrix...
+Rank [29/32]: Calculate diagonal blocks in global distance matrix...
+Rank [11/32]: Calculate diagonal blocks in global distance matrix...
+Rank [16/32]: Start tile-wise ring communication...
+Rank [0/32]: Start tile-wise ring communication...
+Rank [24/32]: Start tile-wise ring communication...
+Rank [8/32]: Start tile-wise ring communication...
+Rank [20/32]: Start tile-wise ring communication...
+Rank [4/32]: Start tile-wise ring communication...
+Rank [30/32]: Start tile-wise ring communication...
+Rank [23/32]: Start tile-wise ring communication...
+Rank [29/32]: Start tile-wise ring communication...
+Rank [19/32]: Start tile-wise ring communication...
+Rank [26/32]: Start tile-wise ring communication...
+Rank [14/32]: Start tile-wise ring communication...
+Rank [1/32]: Start tile-wise ring communication...
+Rank [28/32]: Start tile-wise ring communication...
+Rank [31/32]: Start tile-wise ring communication...
+Rank [7/32]: Start tile-wise ring communication...
+Rank [3/32]: Start tile-wise ring communication...
+Rank [25/32]: Start tile-wise ring communication...
+Rank [12/32]: Start tile-wise ring communication...
+Rank [5/32]: Start tile-wise ring communication...
+Rank [15/32]: Start tile-wise ring communication...
+Rank [13/32]: Start tile-wise ring communication...
+Rank [17/32]: Start tile-wise ring communication...
+Rank [22/32]: Start tile-wise ring communication...
+Rank [6/32]: Start tile-wise ring communication...
+Rank [10/32]: Start tile-wise ring communication...
+Rank [27/32]: Start tile-wise ring communication...
+Rank [21/32]: Start tile-wise ring communication...
+Rank [2/32]: Start tile-wise ring communication...
+Rank [11/32]: Start tile-wise ring communication...
+Rank [9/32]: Start tile-wise ring communication...
+Rank [18/32]: Start tile-wise ring communication...
+Rank [11/32]: [DONE]
+Rank [12/32]: [DONE]
+Rank [14/32]: [DONE]
+Rank [13/32]: [DONE]
+Rank [10/32]: [DONE]
+Rank [9/32]: [DONE]
+Rank [31/32]: [DONE]
+Rank [7/32]: [DONE]
+Rank [23/32]: [DONE]
+Rank [2/32]: [DONE]
+Rank [15/32]: [DONE]
+Rank [0/32]: [DONE]
+Rank [8/32]: [DONE]
+Rank [1/32]: [DONE]
+Rank [6/32]: [DONE]
+Rank [21/32]: [DONE]
+Rank [3/32]: [DONE]
+Rank [22/32]: [DONE]
+Rank [5/32]: [DONE]
+Rank [17/32]: [DONE]
+Rank [18/32]: [DONE]
+Rank [20/32]: [DONE]
+Rank [4/32]: [DONE]
+Rank [16/32]: [DONE]
+Rank [19/32]: [DONE]
+Rank [29/32]: [DONE]
+Rank [26/32]: [DONE]
+Rank [24/32]: [DONE]
+Rank [25/32]: [DONE]
+Rank [27/32]: [DONE]
+Rank [28/32]: [DONE]
+Rank [30/32]: [DONE]
+Rank [0/32]: Local distance matrix has shape [1562, 50000].
+Process-averaged run time: 0.353945613198448 s
+Rank [30/32]: Local distance matrix has shape [1562, 50000].
+Rank [6/32]: Local distance matrix has shape [1562, 50000].
+Rank [28/32]: Local distance matrix has shape [1562, 50000].
+Rank [24/32]: Local distance matrix has shape [1562, 50000].
+Rank [22/32]: Local distance matrix has shape [1562, 50000].
+Rank [23/32]: Local distance matrix has shape [1562, 50000].
+Rank [1/32]: Local distance matrix has shape [1562, 50000].
+Rank [10/32]: Local distance matrix has shape [1562, 50000].
+Rank [27/32]: Local distance matrix has shape [1562, 50000].
+Rank [7/32]: Local distance matrix has shape [1562, 50000].
+Rank [4/32]: Local distance matrix has shape [1562, 50000].
+Rank [3/32]: Local distance matrix has shape [1562, 50000].
+Rank [9/32]: Local distance matrix has shape [1562, 50000].
+Rank [25/32]: Local distance matrix has shape [1562, 50000].
+Rank [12/32]: Local distance matrix has shape [1562, 50000].
+Rank [16/32]: Local distance matrix has shape [1562, 50000].
+Rank [2/32]: Local distance matrix has shape [1562, 50000].
+Rank [5/32]: Local distance matrix has shape [1562, 50000].
+Rank [15/32]: Local distance matrix has shape [1562, 50000].
+Rank [29/32]: Local distance matrix has shape [1562, 50000].
+Rank [11/32]: Local distance matrix has shape [1562, 50000].
+Rank [19/32]: Local distance matrix has shape [1562, 50000].
+Rank [18/32]: Local distance matrix has shape [1562, 50000].
+Rank [13/32]: Local distance matrix has shape [1562, 50000].
+Rank [17/32]: Local distance matrix has shape [1562, 50000].
+Rank [26/32]: Local distance matrix has shape [1562, 50000].
+Rank [14/32]: Local distance matrix has shape [1562, 50000].
+Rank [20/32]: Local distance matrix has shape [1562, 50000].
+Rank [31/32]: Local distance matrix has shape [1578, 50000].
+Rank [8/32]: Local distance matrix has shape [1562, 50000].
+Rank [21/32]: Local distance matrix has shape [1562, 50000].
+Run 8
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 32.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [7/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [0/32]: Local data chunk has shape [1562, 18]...
+Start distance calculations...
+ [OK]
+Rank [13/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [28/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [12/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [16/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [11/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [31/32]: Local data chunk has shape [1578, 18]...
+ [OK]
+Rank [8/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [20/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [3/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [10/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [14/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [2/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [17/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [4/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [15/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [5/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [30/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [1/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [25/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [18/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [6/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [24/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [9/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [27/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [26/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [29/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [23/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [21/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [22/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [19/32]: Local data chunk has shape [1562, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/32]: Calculate diagonal blocks in global distance matrix...
+Rank [8/32]: Calculate diagonal blocks in global distance matrix...
+Rank [17/32]: Calculate diagonal blocks in global distance matrix...
+Rank [22/32]: Calculate diagonal blocks in global distance matrix...
+Rank [29/32]: Calculate diagonal blocks in global distance matrix...
+Rank [24/32]: Calculate diagonal blocks in global distance matrix...
+Rank [18/32]: Calculate diagonal blocks in global distance matrix...
+Rank [19/32]: Calculate diagonal blocks in global distance matrix...
+Rank [23/32]: Calculate diagonal blocks in global distance matrix...
+Rank [11/32]: Calculate diagonal blocks in global distance matrix...
+Rank [12/32]: Calculate diagonal blocks in global distance matrix...
+Rank [16/32]: Calculate diagonal blocks in global distance matrix...
+Rank [5/32]: Calculate diagonal blocks in global distance matrix...
+Rank [20/32]: Calculate diagonal blocks in global distance matrix...
+Rank [27/32]: Calculate diagonal blocks in global distance matrix...
+Rank [4/32]: Calculate diagonal blocks in global distance matrix...
+Rank [15/32]: Calculate diagonal blocks in global distance matrix...
+Rank [21/32]: Calculate diagonal blocks in global distance matrix...
+Rank [28/32]: Calculate diagonal blocks in global distance matrix...
+Rank [25/32]: Calculate diagonal blocks in global distance matrix...
+Rank [31/32]: Calculate diagonal blocks in global distance matrix...
+Rank [2/32]: Calculate diagonal blocks in global distance matrix...
+Rank [7/32]: Calculate diagonal blocks in global distance matrix...
+Rank [6/32]: Calculate diagonal blocks in global distance matrix...
+Rank [30/32]: Calculate diagonal blocks in global distance matrix...
+Rank [9/32]: Calculate diagonal blocks in global distance matrix...
+Rank [13/32]: Calculate diagonal blocks in global distance matrix...
+Rank [3/32]: Calculate diagonal blocks in global distance matrix...
+Rank [1/32]: Calculate diagonal blocks in global distance matrix...
+Rank [26/32]: Calculate diagonal blocks in global distance matrix...
+Rank [10/32]: Calculate diagonal blocks in global distance matrix...
+Rank [14/32]: Calculate diagonal blocks in global distance matrix...
+Rank [0/32]: Start tile-wise ring communication...
+Rank [24/32]: Start tile-wise ring communication...
+Rank [8/32]: Start tile-wise ring communication...
+Rank [16/32]: Start tile-wise ring communication...
+Rank [12/32]: Start tile-wise ring communication...
+Rank [17/32]: Start tile-wise ring communication...
+Rank [21/32]: Start tile-wise ring communication...
+Rank [10/32]: Start tile-wise ring communication...
+Rank [7/32]: Start tile-wise ring communication...
+Rank [14/32]: Start tile-wise ring communication...
+Rank [2/32]: Start tile-wise ring communication...
+Rank [13/32]: Start tile-wise ring communication...
+Rank [29/32]: Start tile-wise ring communication...
+Rank [28/32]: Start tile-wise ring communication...
+Rank [31/32]: Start tile-wise ring communication...
+Rank [20/32]: Start tile-wise ring communication...
+Rank [4/32]: Start tile-wise ring communication...
+Rank [18/32]: Start tile-wise ring communication...
+Rank [27/32]: Start tile-wise ring communication...
+Rank [9/32]: Start tile-wise ring communication...
+Rank [26/32]: Start tile-wise ring communication...
+Rank [3/32]: Start tile-wise ring communication...
+Rank [30/32]: Start tile-wise ring communication...
+Rank [19/32]: Start tile-wise ring communication...
+Rank [11/32]: Start tile-wise ring communication...
+Rank [6/32]: Start tile-wise ring communication...
+Rank [22/32]: Start tile-wise ring communication...
+Rank [15/32]: Start tile-wise ring communication...
+Rank [1/32]: Start tile-wise ring communication...
+Rank [23/32]: Start tile-wise ring communication...
+Rank [5/32]: Start tile-wise ring communication...
+Rank [25/32]: Start tile-wise ring communication...
+Rank [14/32]: [DONE]
+Rank [12/32]: [DONE]
+Rank [7/32]: [DONE]
+Rank [13/32]: [DONE]
+Rank [15/32]: [DONE]
+Rank [22/32]: [DONE]
+Rank [21/32]: [DONE]
+Rank [0/32]: [DONE]
+Rank [31/32]: [DONE]
+Rank [8/32]: [DONE]
+Rank [6/32]: [DONE]
+Rank [2/32]: [DONE]
+Rank [1/32]: [DONE]
+Rank [23/32]: [DONE]
+Rank [11/32]: [DONE]
+Rank [9/32]: [DONE]
+Rank [10/32]: [DONE]
+Rank [18/32]: [DONE]
+Rank [16/32]: [DONE]
+Rank [20/32]: [DONE]
+Rank [19/32]: [DONE]
+Rank [3/32]: [DONE]
+Rank [5/32]: [DONE]
+Rank [4/32]: [DONE]
+Rank [17/32]: [DONE]
+Rank [24/32]: [DONE]
+Rank [29/32]: [DONE]
+Rank [25/32]: [DONE]
+Rank [28/32]: [DONE]
+Rank [26/32]: [DONE]
+Rank [27/32]: [DONE]
+Rank [30/32]: [DONE]
+Rank [0/32]: Local distance matrix has shape [1562, 50000].
+Process-averaged run time: 0.3570676560339052 s
+Rank [30/32]: Local distance matrix has shape [1562, 50000].
+Rank [27/32]: Local distance matrix has shape [1562, 50000].
+Rank [1/32]: Local distance matrix has shape [1562, 50000].
+Rank [25/32]: Local distance matrix has shape [1562, 50000].
+Rank [8/32]: Local distance matrix has shape [1562, 50000].
+Rank [6/32]: Local distance matrix has shape [1562, 50000].
+Rank [17/32]: Local distance matrix has shape [1562, 50000].
+Rank [15/32]: Local distance matrix has shape [1562, 50000].
+Rank [21/32]: Local distance matrix has shape [1562, 50000].
+Rank [3/32]: Local distance matrix has shape [1562, 50000].
+Rank [28/32]: Local distance matrix has shape [1562, 50000].
+Rank [31/32]: Local distance matrix has shape [1578, 50000].
+Rank [24/32]: Local distance matrix has shape [1562, 50000].
+Rank [29/32]: Local distance matrix has shape [1562, 50000].
+Rank [18/32]: Local distance matrix has shape [1562, 50000].
+Rank [26/32]: Local distance matrix has shape [1562, 50000].
+Rank [19/32]: Local distance matrix has shape [1562, 50000].
+Rank [23/32]: Local distance matrix has shape [1562, 50000].
+Rank [11/32]: Local distance matrix has shape [1562, 50000].
+Rank [10/32]: Local distance matrix has shape [1562, 50000].
+Rank [12/32]: Local distance matrix has shape [1562, 50000].
+Rank [14/32]: Local distance matrix has shape [1562, 50000].
+Rank [2/32]: Local distance matrix has shape [1562, 50000].
+Rank [16/32]: Local distance matrix has shape [1562, 50000].
+Rank [5/32]: Local distance matrix has shape [1562, 50000].
+Rank [7/32]: Local distance matrix has shape [1562, 50000].
+Rank [9/32]: Local distance matrix has shape [1562, 50000].
+Rank [20/32]: Local distance matrix has shape [1562, 50000].
+Rank [22/32]: Local distance matrix has shape [1562, 50000].
+Rank [13/32]: Local distance matrix has shape [1562, 50000].
+Rank [4/32]: Local distance matrix has shape [1562, 50000].
+Run 9
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 32.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/32]: Local data chunk has shape [1562, 18]...
+Start distance calculations...
+ [OK]
+Rank [7/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [27/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [19/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [2/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [21/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [23/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [9/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [12/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [10/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [18/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [31/32]: Local data chunk has shape [1578, 18]...
+ [OK]
+Rank [3/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [25/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [17/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [4/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [6/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [15/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [26/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [1/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [28/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [29/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [24/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [16/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [14/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [20/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [22/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [30/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [8/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [13/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [11/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [5/32]: Local data chunk has shape [1562, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/32]: Calculate diagonal blocks in global distance matrix...
+Rank [28/32]: Calculate diagonal blocks in global distance matrix...
+Rank [27/32]: Calculate diagonal blocks in global distance matrix...
+Rank [29/32]: Calculate diagonal blocks in global distance matrix...
+Rank [21/32]: Calculate diagonal blocks in global distance matrix...
+Rank [31/32]: Calculate diagonal blocks in global distance matrix...
+Rank [9/32]: Calculate diagonal blocks in global distance matrix...
+Rank [25/32]: Calculate diagonal blocks in global distance matrix...
+Rank [3/32]: Calculate diagonal blocks in global distance matrix...
+Rank [6/32]: Calculate diagonal blocks in global distance matrix...
+Rank [8/32]: Calculate diagonal blocks in global distance matrix...
+Rank [12/32]: Calculate diagonal blocks in global distance matrix...
+Rank [11/32]: Calculate diagonal blocks in global distance matrix...
+Rank [10/32]: Calculate diagonal blocks in global distance matrix...
+Rank [2/32]: Calculate diagonal blocks in global distance matrix...
+Rank [7/32]: Calculate diagonal blocks in global distance matrix...
+Rank [13/32]: Calculate diagonal blocks in global distance matrix...
+Rank [4/32]: Calculate diagonal blocks in global distance matrix...
+Rank [1/32]: Calculate diagonal blocks in global distance matrix...
+Rank [15/32]: Calculate diagonal blocks in global distance matrix...
+Rank [5/32]: Calculate diagonal blocks in global distance matrix...
+Rank [18/32]: Calculate diagonal blocks in global distance matrix...
+Rank [16/32]: Calculate diagonal blocks in global distance matrix...
+Rank [14/32]: Calculate diagonal blocks in global distance matrix...
+Rank [23/32]: Calculate diagonal blocks in global distance matrix...
+Rank [20/32]: Calculate diagonal blocks in global distance matrix...
+Rank [19/32]: Calculate diagonal blocks in global distance matrix...
+Rank [22/32]: Calculate diagonal blocks in global distance matrix...
+Rank [26/32]: Calculate diagonal blocks in global distance matrix...
+Rank [30/32]: Calculate diagonal blocks in global distance matrix...
+Rank [17/32]: Calculate diagonal blocks in global distance matrix...
+Rank [24/32]: Calculate diagonal blocks in global distance matrix...
+Rank [0/32]: Start tile-wise ring communication...
+Rank [8/32]: Start tile-wise ring communication...
+Rank [16/32]: Start tile-wise ring communication...
+Rank [24/32]: Start tile-wise ring communication...
+Rank [9/32]: Start tile-wise ring communication...
+Rank [31/32]: Start tile-wise ring communication...
+Rank [11/32]: Start tile-wise ring communication...
+Rank [18/32]: Start tile-wise ring communication...
+Rank [14/32]: Start tile-wise ring communication...
+Rank [4/32]: Start tile-wise ring communication...
+Rank [28/32]: Start tile-wise ring communication...
+Rank [20/32]: Start tile-wise ring communication...
+Rank [25/32]: Start tile-wise ring communication...
+Rank [5/32]: Start tile-wise ring communication...
+Rank [27/32]: Start tile-wise ring communication...
+Rank [26/32]: Start tile-wise ring communication...
+Rank [17/32]: Start tile-wise ring communication...
+Rank [29/32]: Start tile-wise ring communication...
+Rank [2/32]: Start tile-wise ring communication...
+Rank [23/32]: Start tile-wise ring communication...
+Rank [12/32]: Start tile-wise ring communication...
+Rank [10/32]: Start tile-wise ring communication...
+Rank [7/32]: Start tile-wise ring communication...
+Rank [19/32]: Start tile-wise ring communication...
+Rank [22/32]: Start tile-wise ring communication...
+Rank [6/32]: Start tile-wise ring communication...
+Rank [13/32]: Start tile-wise ring communication...
+Rank [1/32]: Start tile-wise ring communication...
+Rank [30/32]: Start tile-wise ring communication...
+Rank [21/32]: Start tile-wise ring communication...
+Rank [15/32]: Start tile-wise ring communication...
+Rank [3/32]: Start tile-wise ring communication...
+Rank [0/32]: [DONE]
+Rank [23/32]: [DONE]
+Rank [31/32]: [DONE]
+Rank [2/32]: [DONE]
+Rank [1/32]: [DONE]
+Rank [14/32]: [DONE]
+Rank [12/32]: [DONE]
+Rank [3/32]: [DONE]
+Rank [21/32]: [DONE]
+Rank [7/32]: [DONE]
+Rank [13/32]: [DONE]
+Rank [5/32]: [DONE]
+Rank [4/32]: [DONE]
+Rank [6/32]: [DONE]
+Rank [15/32]: [DONE]
+Rank [22/32]: [DONE]
+Rank [11/32]: [DONE]
+Rank [8/32]: [DONE]
+Rank [27/32]: [DONE]
+Rank [9/32]: [DONE]
+Rank [10/32]: [DONE]
+Rank [18/32]: [DONE]
+Rank [20/32]: [DONE]
+Rank [28/32]: [DONE]
+Rank [19/32]: [DONE]
+Rank [17/32]: [DONE]
+Rank [29/32]: [DONE]
+Rank [26/32]: [DONE]
+Rank [16/32]: [DONE]
+Rank [24/32]: [DONE]
+Rank [25/32]: [DONE]
+Rank [0/32]: Local distance matrix has shape [1562, 50000].
+Process-averaged run time: 0.35246020174236037 s
+Rank [30/32]: [DONE]
+Rank [30/32]: Local distance matrix has shape [1562, 50000].
+Rank [5/32]: Local distance matrix has shape [1562, 50000].
+Rank [16/32]: Local distance matrix has shape [1562, 50000].
+Rank [14/32]: Local distance matrix has shape [1562, 50000].
+Rank [25/32]: Local distance matrix has shape [1562, 50000].
+Rank [23/32]: Local distance matrix has shape [1562, 50000].
+Rank [27/32]: Local distance matrix has shape [1562, 50000].
+Rank [3/32]: Local distance matrix has shape [1562, 50000].
+Rank [6/32]: Local distance matrix has shape [1562, 50000].
+Rank [12/32]: Local distance matrix has shape [1562, 50000].
+Rank [11/32]: Local distance matrix has shape [1562, 50000].
+Rank [17/32]: Local distance matrix has shape [1562, 50000].
+Rank [24/32]: Local distance matrix has shape [1562, 50000].
+Rank [29/32]: Local distance matrix has shape [1562, 50000].
+Rank [21/32]: Local distance matrix has shape [1562, 50000].
+Rank [31/32]: Local distance matrix has shape [1578, 50000].
+Rank [8/32]: Local distance matrix has shape [1562, 50000].
+Rank [10/32]: Local distance matrix has shape [1562, 50000].
+Rank [2/32]: Local distance matrix has shape [1562, 50000].
+Rank [7/32]: Local distance matrix has shape [1562, 50000].
+Rank [13/32]: Local distance matrix has shape [1562, 50000].
+Rank [4/32]: Local distance matrix has shape [1562, 50000].
+Rank [1/32]: Local distance matrix has shape [1562, 50000].
+Rank [15/32]: Local distance matrix has shape [1562, 50000].
+Rank [9/32]: Local distance matrix has shape [1562, 50000].
+Rank [18/32]: Local distance matrix has shape [1562, 50000].
+Rank [28/32]: Local distance matrix has shape [1562, 50000].
+Rank [20/32]: Local distance matrix has shape [1562, 50000].
+Rank [19/32]: Local distance matrix has shape [1562, 50000].
+Rank [22/32]: Local distance matrix has shape [1562, 50000].
+Rank [26/32]: Local distance matrix has shape [1562, 50000].
+Run 10
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 32.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [30/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [21/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [4/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [31/32]: Local data chunk has shape [1578, 18]...
+ [OK]
+Rank [22/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [15/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [6/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [25/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [8/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [18/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [5/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [9/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [12/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [29/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [13/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [7/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [1/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [20/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [28/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [3/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [24/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [26/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [19/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [10/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [14/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [2/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [11/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [27/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [0/32]: Local data chunk has shape [1562, 18]...
+Start distance calculations...
+ [OK]
+Rank [17/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [23/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [16/32]: Local data chunk has shape [1562, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/32]: Calculate diagonal blocks in global distance matrix...
+Rank [26/32]: Calculate diagonal blocks in global distance matrix...
+Rank [24/32]: Calculate diagonal blocks in global distance matrix...
+Rank [2/32]: Calculate diagonal blocks in global distance matrix...
+Rank [28/32]: Calculate diagonal blocks in global distance matrix...
+Rank [1/32]: Calculate diagonal blocks in global distance matrix...
+Rank [21/32]: Calculate diagonal blocks in global distance matrix...
+Rank [30/32]: Calculate diagonal blocks in global distance matrix...
+Rank [7/32]: Calculate diagonal blocks in global distance matrix...
+Rank [25/32]: Calculate diagonal blocks in global distance matrix...
+Rank [17/32]: Calculate diagonal blocks in global distance matrix...
+Rank [20/32]: Calculate diagonal blocks in global distance matrix...
+Rank [29/32]: Calculate diagonal blocks in global distance matrix...
+Rank [8/32]: Calculate diagonal blocks in global distance matrix...
+Rank [15/32]: Calculate diagonal blocks in global distance matrix...
+Rank [9/32]: Calculate diagonal blocks in global distance matrix...
+Rank [18/32]: Calculate diagonal blocks in global distance matrix...
+Rank [19/32]: Calculate diagonal blocks in global distance matrix...
+Rank [16/32]: Calculate diagonal blocks in global distance matrix...
+Rank [23/32]: Calculate diagonal blocks in global distance matrix...
+Rank [4/32]: Calculate diagonal blocks in global distance matrix...
+Rank [27/32]: Calculate diagonal blocks in global distance matrix...
+Rank [3/32]: Calculate diagonal blocks in global distance matrix...
+Rank [11/32]: Calculate diagonal blocks in global distance matrix...
+Rank [31/32]: Calculate diagonal blocks in global distance matrix...
+Rank [12/32]: Calculate diagonal blocks in global distance matrix...
+Rank [14/32]: Calculate diagonal blocks in global distance matrix...
+Rank [5/32]: Calculate diagonal blocks in global distance matrix...
+Rank [22/32]: Calculate diagonal blocks in global distance matrix...
+Rank [10/32]: Calculate diagonal blocks in global distance matrix...
+Rank [6/32]: Calculate diagonal blocks in global distance matrix...
+Rank [13/32]: Calculate diagonal blocks in global distance matrix...
+Rank [8/32]: Start tile-wise ring communication...
+Rank [0/32]: Start tile-wise ring communication...
+Rank [16/32]: Start tile-wise ring communication...
+Rank [24/32]: Start tile-wise ring communication...
+Rank [11/32]: Start tile-wise ring communication...
+Rank [12/32]: Start tile-wise ring communication...
+Rank [30/32]: Start tile-wise ring communication...
+Rank [1/32]: Start tile-wise ring communication...
+Rank [15/32]: Start tile-wise ring communication...
+Rank [9/32]: Start tile-wise ring communication...
+Rank [18/32]: Start tile-wise ring communication...
+Rank [4/32]: Start tile-wise ring communication...
+Rank [22/32]: Start tile-wise ring communication...
+Rank [7/32]: Start tile-wise ring communication...
+Rank [25/32]: Start tile-wise ring communication...
+Rank [23/32]: Start tile-wise ring communication...
+Rank [31/32]: Start tile-wise ring communication...
+Rank [26/32]: Start tile-wise ring communication...
+Rank [19/32]: Start tile-wise ring communication...
+Rank [27/32]: Start tile-wise ring communication...
+Rank [3/32]: Start tile-wise ring communication...
+Rank [2/32]: Start tile-wise ring communication...
+Rank [5/32]: Start tile-wise ring communication...
+Rank [28/32]: Start tile-wise ring communication...
+Rank [10/32]: Start tile-wise ring communication...
+Rank [6/32]: Start tile-wise ring communication...
+Rank [13/32]: Start tile-wise ring communication...
+Rank [17/32]: Start tile-wise ring communication...
+Rank [21/32]: Start tile-wise ring communication...
+Rank [20/32]: Start tile-wise ring communication...
+Rank [29/32]: Start tile-wise ring communication...
+Rank [14/32]: Start tile-wise ring communication...
+Rank [15/32]: [DONE]
+Rank [0/32]: [DONE]
+Rank [1/32]: [DONE]
+Rank [2/32]: [DONE]
+Rank [14/32]: [DONE]
+Rank [13/32]: [DONE]
+Rank [16/32]: [DONE]
+Rank [23/32]: [DONE]
+Rank [17/32]: [DONE]
+Rank [31/32]: [DONE]
+Rank [12/32]: [DONE]
+Rank [11/32]: [DONE]
+Rank [21/32]: [DONE]
+Rank [10/32]: [DONE]
+Rank [24/32]: [DONE]
+Rank [3/32]: [DONE]
+Rank [22/32]: [DONE]
+Rank [5/32]: [DONE]
+Rank [18/32]: [DONE]
+Rank [20/32]: [DONE]
+Rank [7/32]: [DONE]
+Rank [19/32]: [DONE]
+Rank [4/32]: [DONE]
+Rank [9/32]: [DONE]
+Rank [6/32]: [DONE]
+Rank [8/32]: [DONE]
+Rank [26/32]: [DONE]
+Rank [27/32]: [DONE]
+Rank [25/32]: [DONE]
+Rank [29/32]: [DONE]
+Rank [30/32]: [DONE]
+Rank [0/32]: Local distance matrix has shape [1562, 50000].
+Process-averaged run time: 0.35467995997169055 s
+Rank [28/32]: [DONE]
+Rank [13/32]: Local distance matrix has shape [1562, 50000].
+Rank [7/32]: Local distance matrix has shape [1562, 50000].
+Rank [1/32]: Local distance matrix has shape [1562, 50000].
+Rank [18/32]: Local distance matrix has shape [1562, 50000].
+Rank [4/32]: Local distance matrix has shape [1562, 50000].
+Rank [27/32]: Local distance matrix has shape [1562, 50000].
+Rank [12/32]: Local distance matrix has shape [1562, 50000].
+Rank [5/32]: Local distance matrix has shape [1562, 50000].
+Rank [28/32]: Local distance matrix has shape [1562, 50000].
+Rank [10/32]: Local distance matrix has shape [1562, 50000].
+Rank [17/32]: Local distance matrix has shape [1562, 50000].
+Rank [21/32]: Local distance matrix has shape [1562, 50000].
+Rank [20/32]: Local distance matrix has shape [1562, 50000].
+Rank [29/32]: Local distance matrix has shape [1562, 50000].
+Rank [26/32]: Local distance matrix has shape [1562, 50000].
+Rank [8/32]: Local distance matrix has shape [1562, 50000].
+Rank [15/32]: Local distance matrix has shape [1562, 50000].
+Rank [9/32]: Local distance matrix has shape [1562, 50000].
+Rank [19/32]: Local distance matrix has shape [1562, 50000].
+Rank [16/32]: Local distance matrix has shape [1562, 50000].
+Rank [23/32]: Local distance matrix has shape [1562, 50000].
+Rank [24/32]: Local distance matrix has shape [1562, 50000].
+Rank [3/32]: Local distance matrix has shape [1562, 50000].
+Rank [11/32]: Local distance matrix has shape [1562, 50000].
+Rank [2/32]: Local distance matrix has shape [1562, 50000].
+Rank [31/32]: Local distance matrix has shape [1578, 50000].
+Rank [14/32]: Local distance matrix has shape [1562, 50000].
+Rank [22/32]: Local distance matrix has shape [1562, 50000].
+Rank [6/32]: Local distance matrix has shape [1562, 50000].
+Rank [30/32]: Local distance matrix has shape [1562, 50000].
+Rank [25/32]: Local distance matrix has shape [1562, 50000].
+Run 11
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 32.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/32]: Local data chunk has shape [1562, 18]...
+Start distance calculations...
+ [OK]
+Rank [27/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [24/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [18/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [11/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [15/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [31/32]: Local data chunk has shape [1578, 18]...
+ [OK]
+Rank [25/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [13/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [1/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [9/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [4/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [22/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [2/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [16/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [17/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [3/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [19/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [21/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [30/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [14/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [12/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [6/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [7/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [23/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [29/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [5/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [28/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [10/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [20/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [26/32]: Local data chunk has shape [1562, 18]...
+ [OK]
+Rank [8/32]: Local data chunk has shape [1562, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/32]: Calculate diagonal blocks in global distance matrix...
+Rank [2/32]: Calculate diagonal blocks in global distance matrix...
+Rank [6/32]: Calculate diagonal blocks in global distance matrix...
+Rank [27/32]: Calculate diagonal blocks in global distance matrix...
+Rank [8/32]: Calculate diagonal blocks in global distance matrix...
+Rank [4/32]: Calculate diagonal blocks in global distance matrix...
+Rank [7/32]: Calculate diagonal blocks in global distance matrix...
+Rank [31/32]: Calculate diagonal blocks in global distance matrix...
+Rank [5/32]: Calculate diagonal blocks in global distance matrix...
+Rank [25/32]: Calculate diagonal blocks in global distance matrix...
+Rank [10/32]: Calculate diagonal blocks in global distance matrix...
+Rank [23/32]: Calculate diagonal blocks in global distance matrix...
+Rank [16/32]: Calculate diagonal blocks in global distance matrix...
+Rank [29/32]: Calculate diagonal blocks in global distance matrix...
+Rank [20/32]: Calculate diagonal blocks in global distance matrix...
+Rank [17/32]: Calculate diagonal blocks in global distance matrix...
+Rank [3/32]: Calculate diagonal blocks in global distance matrix...
+Rank [13/32]: Calculate diagonal blocks in global distance matrix...
+Rank [1/32]: Calculate diagonal blocks in global distance matrix...
+Rank [19/32]: Calculate diagonal blocks in global distance matrix...
+Rank [26/32]: Calculate diagonal blocks in global distance matrix...
+Rank [24/32]: Calculate diagonal blocks in global distance matrix...
+Rank [21/32]: Calculate diagonal blocks in global distance matrix...
+Rank [30/32]: Calculate diagonal blocks in global distance matrix...
+Rank [14/32]: Calculate diagonal blocks in global distance matrix...
+Rank [9/32]: Calculate diagonal blocks in global distance matrix...
+Rank [18/32]: Calculate diagonal blocks in global distance matrix...
+Rank [22/32]: Calculate diagonal blocks in global distance matrix...
+Rank [28/32]: Calculate diagonal blocks in global distance matrix...
+Rank [12/32]: Calculate diagonal blocks in global distance matrix...
+Rank [15/32]: Calculate diagonal blocks in global distance matrix...
+Rank [11/32]: Calculate diagonal blocks in global distance matrix...
+Rank [8/32]: Start tile-wise ring communication...
+Rank [0/32]: Start tile-wise ring communication...
+Rank [24/32]: Start tile-wise ring communication...
+Rank [16/32]: Start tile-wise ring communication...
+Rank [7/32]: Start tile-wise ring communication...
+Rank [9/32]: Start tile-wise ring communication...
+Rank [21/32]: Start tile-wise ring communication...
+Rank [31/32]: Start tile-wise ring communication...
+Rank [1/32]: Start tile-wise ring communication...
+Rank [29/32]: Start tile-wise ring communication...
+Rank [14/32]: Start tile-wise ring communication...
+Rank [28/32]: Start tile-wise ring communication...
+Rank [10/32]: Start tile-wise ring communication...
+Rank [4/32]: Start tile-wise ring communication...
+Rank [17/32]: Start tile-wise ring communication...
+Rank [13/32]: Start tile-wise ring communication...
+Rank [18/32]: Start tile-wise ring communication...
+Rank [2/32]: Start tile-wise ring communication...
+Rank [27/32]: Start tile-wise ring communication...
+Rank [15/32]: Start tile-wise ring communication...
+Rank [20/32]: Start tile-wise ring communication...
+Rank [22/32]: Start tile-wise ring communication...
+Rank [12/32]: Start tile-wise ring communication...
+Rank [3/32]: Start tile-wise ring communication...
+Rank [6/32]: Start tile-wise ring communication...
+Rank [25/32]: Start tile-wise ring communication...
+Rank [23/32]: Start tile-wise ring communication...
+Rank [30/32]: Start tile-wise ring communication...
+Rank [26/32]: Start tile-wise ring communication...
+Rank [19/32]: Start tile-wise ring communication...
+Rank [11/32]: Start tile-wise ring communication...
+Rank [5/32]: Start tile-wise ring communication...
+Rank [0/32]: [DONE]
+Rank [2/32]: [DONE]
+Rank [31/32]: [DONE]
+Rank [1/32]: [DONE]
+Rank [15/32]: [DONE]
+Rank [12/32]: [DONE]
+Rank [7/32]: [DONE]
+Rank [14/32]: [DONE]
+Rank [23/32]: [DONE]
+Rank [21/32]: [DONE]
+Rank [18/32]: [DONE]
+Rank [13/32]: [DONE]
+Rank [20/32]: [DONE]
+Rank [22/32]: [DONE]
+Rank [5/32]: [DONE]
+Rank [3/32]: [DONE]
+Rank [6/32]: [DONE]
+Rank [8/32]: [DONE]
+Rank [19/32]: [DONE]
+Rank [9/32]: [DONE]
+Rank [4/32]: [DONE]
+Rank [11/32]: [DONE]
+Rank [17/32]: [DONE]
+Rank [10/32]: [DONE]
+Rank [16/32]: [DONE]
+Rank [26/32]: [DONE]
+Rank [27/32]: [DONE]
+Rank [28/32]: [DONE]
+Rank [24/32]: [DONE]
+Rank [25/32]: [DONE]
+Rank [29/32]: [DONE]
+Rank [0/32]: Local distance matrix has shape [1562, 50000].
+Process-averaged run time: 0.3460264903551433 s
+Rank [30/32]: [DONE]
+Rank [25/32]: Local distance matrix has shape [1562, 50000].
+Rank [30/32]: Local distance matrix has shape [1562, 50000].
+Rank [28/32]: Local distance matrix has shape [1562, 50000].
+Rank [20/32]: Local distance matrix has shape [1562, 50000].
+Rank [17/32]: Local distance matrix has shape [1562, 50000].
+Rank [13/32]: Local distance matrix has shape [1562, 50000].
+Rank [11/32]: Local distance matrix has shape [1562, 50000].
+Rank [27/32]: Local distance matrix has shape [1562, 50000].
+Rank [10/32]: Local distance matrix has shape [1562, 50000].
+Rank [23/32]: Local distance matrix has shape [1562, 50000].
+Rank [16/32]: Local distance matrix has shape [1562, 50000].
+Rank [29/32]: Local distance matrix has shape [1562, 50000].
+Rank [8/32]: Local distance matrix has shape [1562, 50000].
+Rank [3/32]: Local distance matrix has shape [1562, 50000].
+Rank [1/32]: Local distance matrix has shape [1562, 50000].
+Rank [19/32]: Local distance matrix has shape [1562, 50000].
+Rank [26/32]: Local distance matrix has shape [1562, 50000].
+Rank [24/32]: Local distance matrix has shape [1562, 50000].
+Rank [14/32]: Local distance matrix has shape [1562, 50000].
+Rank [18/32]: Local distance matrix has shape [1562, 50000].
+Rank [4/32]: Local distance matrix has shape [1562, 50000].
+Rank [2/32]: Local distance matrix has shape [1562, 50000].
+Rank [15/32]: Local distance matrix has shape [1562, 50000].
+Rank [7/32]: Local distance matrix has shape [1562, 50000].
+Rank [31/32]: Local distance matrix has shape [1578, 50000].
+Rank [5/32]: Local distance matrix has shape [1562, 50000].
+Rank [21/32]: Local distance matrix has shape [1562, 50000].
+Rank [9/32]: Local distance matrix has shape [1562, 50000].
+Rank [22/32]: Local distance matrix has shape [1562, 50000].
+Rank [12/32]: Local distance matrix has shape [1562, 50000].
+Rank [6/32]: Local distance matrix has shape [1562, 50000].
+
+============================= JOB FEEDBACK =============================
+
+NodeName=uc2n[051-080,171-172]
+Job ID: 24678136
+Cluster: uc2
+User/Group: ku4408/scc
+State: COMPLETED (exit code 0)
+Nodes: 32
+Cores per node: 80
+CPU Utilized: 00:15:27
+CPU Efficiency: 0.22% of 4-19:54:40 core-walltime
+Job Wall-clock time: 00:02:43
+Memory Utilized: 484.40 MB
+Memory Efficiency: 0.02% of 2.75 TB
diff --git a/2_psrs_cdist/solutions/A1/strong_scaling/nodes_4.out b/2_psrs_cdist/solutions/A1/strong_scaling/nodes_4.out
new file mode 100644
index 0000000..9e6fea4
--- /dev/null
+++ b/2_psrs_cdist/solutions/A1/strong_scaling/nodes_4.out
@@ -0,0 +1,411 @@
+Run 1
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [3/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [0/4]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+ [OK]
+Rank [2/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [1/4]: Local data chunk has shape [12500, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [3/4]: Calculate diagonal blocks in global distance matrix...
+Rank [1/4]: Calculate diagonal blocks in global distance matrix...
+Rank [0/4]: Calculate diagonal blocks in global distance matrix...
+Rank [2/4]: Calculate diagonal blocks in global distance matrix...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [3/4]: Start tile-wise ring communication...
+Rank [2/4]: Start tile-wise ring communication...
+Rank [1/4]: Start tile-wise ring communication...
+Rank [3/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [2/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [12500, 50000].
+Process-averaged run time: 2.2419050564058125 s
+Rank [2/4]: Local distance matrix has shape [12500, 50000].
+Rank [1/4]: Local distance matrix has shape [12500, 50000].
+Rank [3/4]: Local distance matrix has shape [12500, 50000].
+Run 2
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [2/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [3/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [1/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [0/4]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/4]: Calculate diagonal blocks in global distance matrix...
+Rank [2/4]: Calculate diagonal blocks in global distance matrix...
+Rank [3/4]: Calculate diagonal blocks in global distance matrix...
+Rank [1/4]: Calculate diagonal blocks in global distance matrix...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [3/4]: Start tile-wise ring communication...
+Rank [2/4]: Start tile-wise ring communication...
+Rank [1/4]: Start tile-wise ring communication...
+Rank [3/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [2/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [12500, 50000].
+Process-averaged run time: 2.143836056580767 s
+Rank [2/4]: Local distance matrix has shape [12500, 50000].
+Rank [3/4]: Local distance matrix has shape [12500, 50000].
+Rank [1/4]: Local distance matrix has shape [12500, 50000].
+Run 3
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/4]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+ [OK]
+Rank [2/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [3/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [1/4]: Local data chunk has shape [12500, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/4]: Calculate diagonal blocks in global distance matrix...
+Rank [2/4]: Calculate diagonal blocks in global distance matrix...
+Rank [1/4]: Calculate diagonal blocks in global distance matrix...
+Rank [3/4]: Calculate diagonal blocks in global distance matrix...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [2/4]: Start tile-wise ring communication...
+Rank [1/4]: Start tile-wise ring communication...
+Rank [3/4]: Start tile-wise ring communication...
+Rank [3/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [2/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [12500, 50000].
+Process-averaged run time: 2.1328196288086474 s
+Rank [1/4]: Local distance matrix has shape [12500, 50000].
+Rank [3/4]: Local distance matrix has shape [12500, 50000].
+Rank [2/4]: Local distance matrix has shape [12500, 50000].
+Run 4
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [3/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [0/4]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [2/4]: Local data chunk has shape [12500, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/4]: Calculate diagonal blocks in global distance matrix...
+Rank [2/4]: Calculate diagonal blocks in global distance matrix...
+Rank [3/4]: Calculate diagonal blocks in global distance matrix...
+Rank [1/4]: Calculate diagonal blocks in global distance matrix...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [2/4]: Start tile-wise ring communication...
+Rank [3/4]: Start tile-wise ring communication...
+Rank [1/4]: Start tile-wise ring communication...
+Rank [3/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [2/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [12500, 50000].
+Process-averaged run time: 2.1412369539029896 s
+Rank [3/4]: Local distance matrix has shape [12500, 50000].
+Rank [2/4]: Local distance matrix has shape [12500, 50000].
+Rank [1/4]: Local distance matrix has shape [12500, 50000].
+Run 5
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [2/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [1/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [0/4]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+ [OK]
+Rank [3/4]: Local data chunk has shape [12500, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/4]: Calculate diagonal blocks in global distance matrix...
+Rank [1/4]: Calculate diagonal blocks in global distance matrix...
+Rank [3/4]: Calculate diagonal blocks in global distance matrix...
+Rank [2/4]: Calculate diagonal blocks in global distance matrix...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [3/4]: Start tile-wise ring communication...
+Rank [2/4]: Start tile-wise ring communication...
+Rank [1/4]: Start tile-wise ring communication...
+Rank [3/4]: [DONE]
+Rank [2/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [12500, 50000].
+Process-averaged run time: 2.1448243020568043 s
+Rank [3/4]: Local distance matrix has shape [12500, 50000].
+Rank [2/4]: Local distance matrix has shape [12500, 50000].
+Rank [1/4]: Local distance matrix has shape [12500, 50000].
+Run 6
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [2/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [1/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [0/4]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+ [OK]
+Rank [3/4]: Local data chunk has shape [12500, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/4]: Calculate diagonal blocks in global distance matrix...
+Rank [2/4]: Calculate diagonal blocks in global distance matrix...
+Rank [3/4]: Calculate diagonal blocks in global distance matrix...
+Rank [1/4]: Calculate diagonal blocks in global distance matrix...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [2/4]: Start tile-wise ring communication...
+Rank [3/4]: Start tile-wise ring communication...
+Rank [1/4]: Start tile-wise ring communication...
+Rank [3/4]: [DONE]
+Rank [2/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [12500, 50000].
+Process-averaged run time: 2.1342911082319915 s
+Rank [2/4]: Local distance matrix has shape [12500, 50000].
+Rank [3/4]: Local distance matrix has shape [12500, 50000].
+Rank [1/4]: Local distance matrix has shape [12500, 50000].
+Run 7
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/4]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+ [OK]
+Rank [2/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [3/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [1/4]: Local data chunk has shape [12500, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/4]: Calculate diagonal blocks in global distance matrix...
+Rank [2/4]: Calculate diagonal blocks in global distance matrix...
+Rank [3/4]: Calculate diagonal blocks in global distance matrix...
+Rank [1/4]: Calculate diagonal blocks in global distance matrix...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [2/4]: Start tile-wise ring communication...
+Rank [1/4]: Start tile-wise ring communication...
+Rank [3/4]: Start tile-wise ring communication...
+Rank [3/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [2/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [12500, 50000].
+Process-averaged run time: 2.131936827208847 s
+Rank [1/4]: Local distance matrix has shape [12500, 50000].
+Rank [3/4]: Local distance matrix has shape [12500, 50000].
+Rank [2/4]: Local distance matrix has shape [12500, 50000].
+Run 8
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [3/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [1/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [2/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [0/4]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/4]: Calculate diagonal blocks in global distance matrix...
+Rank [2/4]: Calculate diagonal blocks in global distance matrix...
+Rank [3/4]: Calculate diagonal blocks in global distance matrix...
+Rank [1/4]: Calculate diagonal blocks in global distance matrix...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [2/4]: Start tile-wise ring communication...
+Rank [1/4]: Start tile-wise ring communication...
+Rank [3/4]: Start tile-wise ring communication...
+Rank [3/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [2/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [12500, 50000].
+Process-averaged run time: 2.134611321846023 s
+Rank [1/4]: Local distance matrix has shape [12500, 50000].
+Rank [2/4]: Local distance matrix has shape [12500, 50000].
+Rank [3/4]: Local distance matrix has shape [12500, 50000].
+Run 9
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/4]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [2/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [3/4]: Local data chunk has shape [12500, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/4]: Calculate diagonal blocks in global distance matrix...
+Rank [2/4]: Calculate diagonal blocks in global distance matrix...
+Rank [3/4]: Calculate diagonal blocks in global distance matrix...
+Rank [1/4]: Calculate diagonal blocks in global distance matrix...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [2/4]: Start tile-wise ring communication...
+Rank [3/4]: Start tile-wise ring communication...
+Rank [1/4]: Start tile-wise ring communication...
+Rank [3/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [2/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [12500, 50000].
+Process-averaged run time: 2.1346833237912506 s
+Rank [3/4]: Local distance matrix has shape [12500, 50000].
+Rank [2/4]: Local distance matrix has shape [12500, 50000].
+Rank [1/4]: Local distance matrix has shape [12500, 50000].
+Run 10
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [3/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [1/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [0/4]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+ [OK]
+Rank [2/4]: Local data chunk has shape [12500, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/4]: Calculate diagonal blocks in global distance matrix...
+Rank [3/4]: Calculate diagonal blocks in global distance matrix...
+Rank [1/4]: Calculate diagonal blocks in global distance matrix...
+Rank [2/4]: Calculate diagonal blocks in global distance matrix...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [3/4]: Start tile-wise ring communication...
+Rank [2/4]: Start tile-wise ring communication...
+Rank [1/4]: Start tile-wise ring communication...
+Rank [3/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [2/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [12500, 50000].
+Process-averaged run time: 2.1418009700719267 s
+Rank [3/4]: Local distance matrix has shape [12500, 50000].
+Rank [1/4]: Local distance matrix has shape [12500, 50000].
+Rank [2/4]: Local distance matrix has shape [12500, 50000].
+Run 11
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/4]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+ [OK]
+Rank [2/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [1/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [3/4]: Local data chunk has shape [12500, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/4]: Calculate diagonal blocks in global distance matrix...
+Rank [3/4]: Calculate diagonal blocks in global distance matrix...
+Rank [2/4]: Calculate diagonal blocks in global distance matrix...
+Rank [1/4]: Calculate diagonal blocks in global distance matrix...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [2/4]: Start tile-wise ring communication...
+Rank [3/4]: Start tile-wise ring communication...
+Rank [1/4]: Start tile-wise ring communication...
+Rank [3/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [2/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [12500, 50000].
+Process-averaged run time: 2.141862073680386 s
+Rank [2/4]: Local distance matrix has shape [12500, 50000].
+Rank [3/4]: Local distance matrix has shape [12500, 50000].
+Rank [1/4]: Local distance matrix has shape [12500, 50000].
+
+============================= JOB FEEDBACK =============================
+
+NodeName=uc2n[233-234,236-237]
+Job ID: 24674016
+Cluster: uc2
+User/Group: ku4408/scc
+State: COMPLETED (exit code 0)
+Nodes: 4
+Cores per node: 80
+CPU Utilized: 00:03:11
+CPU Efficiency: 0.52% of 10:08:00 core-walltime
+Job Wall-clock time: 00:01:54
+Memory Utilized: 3.98 GB
+Memory Efficiency: 1.13% of 351.56 GB
diff --git a/2_psrs_cdist/solutions/A1/strong_scaling/nodes_8.out b/2_psrs_cdist/solutions/A1/strong_scaling/nodes_8.out
new file mode 100644
index 0000000..f5f702f
--- /dev/null
+++ b/2_psrs_cdist/solutions/A1/strong_scaling/nodes_8.out
@@ -0,0 +1,719 @@
+Run 1
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [3/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [0/8]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [4/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [5/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [5/8]: Calculate diagonal blocks in global distance matrix...
+Rank [1/8]: Calculate diagonal blocks in global distance matrix...
+Rank [2/8]: Calculate diagonal blocks in global distance matrix...
+Rank [3/8]: Calculate diagonal blocks in global distance matrix...
+Rank [6/8]: Calculate diagonal blocks in global distance matrix...
+Rank [4/8]: Calculate diagonal blocks in global distance matrix...
+Rank [7/8]: Calculate diagonal blocks in global distance matrix...
+Rank [0/8]: Calculate diagonal blocks in global distance matrix...
+Rank [0/8]: Start tile-wise ring communication...
+Rank [5/8]: Start tile-wise ring communication...
+Rank [1/8]: Start tile-wise ring communication...
+Rank [3/8]: Start tile-wise ring communication...
+Rank [6/8]: Start tile-wise ring communication...
+Rank [2/8]: Start tile-wise ring communication...
+Rank [7/8]: Start tile-wise ring communication...
+Rank [4/8]: Start tile-wise ring communication...
+Rank [7/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [6/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [6250, 50000].
+Process-averaged run time: 1.2568223926937208 s
+Rank [2/8]: Local distance matrix has shape [6250, 50000].
+Rank [1/8]: Local distance matrix has shape [6250, 50000].
+Rank [4/8]: Local distance matrix has shape [6250, 50000].
+Rank [7/8]: Local distance matrix has shape [6250, 50000].
+Rank [5/8]: Local distance matrix has shape [6250, 50000].
+Rank [3/8]: Local distance matrix has shape [6250, 50000].
+Rank [6/8]: Local distance matrix has shape [6250, 50000].
+Run 2
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [6/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [5/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [4/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [3/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [0/8]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [2/8]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/8]: Calculate diagonal blocks in global distance matrix...
+Rank [5/8]: Calculate diagonal blocks in global distance matrix...
+Rank [6/8]: Calculate diagonal blocks in global distance matrix...
+Rank [3/8]: Calculate diagonal blocks in global distance matrix...
+Rank [1/8]: Calculate diagonal blocks in global distance matrix...
+Rank [2/8]: Calculate diagonal blocks in global distance matrix...
+Rank [4/8]: Calculate diagonal blocks in global distance matrix...
+Rank [7/8]: Calculate diagonal blocks in global distance matrix...
+Rank [0/8]: Start tile-wise ring communication...
+Rank [3/8]: Start tile-wise ring communication...
+Rank [2/8]: Start tile-wise ring communication...
+Rank [6/8]: Start tile-wise ring communication...
+Rank [4/8]: Start tile-wise ring communication...
+Rank [7/8]: Start tile-wise ring communication...
+Rank [1/8]: Start tile-wise ring communication...
+Rank [5/8]: Start tile-wise ring communication...
+Rank [7/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [6/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [6250, 50000].
+Process-averaged run time: 1.0980603359639645 s
+Rank [5/8]: Local distance matrix has shape [6250, 50000].
+Rank [7/8]: Local distance matrix has shape [6250, 50000].
+Rank [2/8]: Local distance matrix has shape [6250, 50000].
+Rank [4/8]: Local distance matrix has shape [6250, 50000].
+Rank [1/8]: Local distance matrix has shape [6250, 50000].
+Rank [6/8]: Local distance matrix has shape [6250, 50000].
+Rank [3/8]: Local distance matrix has shape [6250, 50000].
+Run 3
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/8]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [5/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [4/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [3/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/8]: Calculate diagonal blocks in global distance matrix...
+Rank [3/8]: Calculate diagonal blocks in global distance matrix...
+Rank [6/8]: Calculate diagonal blocks in global distance matrix...
+Rank [2/8]: Calculate diagonal blocks in global distance matrix...
+Rank [4/8]: Calculate diagonal blocks in global distance matrix...
+Rank [1/8]: Calculate diagonal blocks in global distance matrix...
+Rank [5/8]: Calculate diagonal blocks in global distance matrix...
+Rank [7/8]: Calculate diagonal blocks in global distance matrix...
+Rank [0/8]: Start tile-wise ring communication...
+Rank [3/8]: Start tile-wise ring communication...
+Rank [6/8]: Start tile-wise ring communication...
+Rank [7/8]: Start tile-wise ring communication...
+Rank [2/8]: Start tile-wise ring communication...
+Rank [1/8]: Start tile-wise ring communication...
+Rank [5/8]: Start tile-wise ring communication...
+Rank [4/8]: Start tile-wise ring communication...
+Rank [7/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [6/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [6250, 50000].
+Process-averaged run time: 1.0971945124911144 s
+Rank [6/8]: Local distance matrix has shape [6250, 50000].
+Rank [5/8]: Local distance matrix has shape [6250, 50000].
+Rank [3/8]: Local distance matrix has shape [6250, 50000].
+Rank [4/8]: Local distance matrix has shape [6250, 50000].
+Rank [2/8]: Local distance matrix has shape [6250, 50000].
+Rank [7/8]: Local distance matrix has shape [6250, 50000].
+Rank [1/8]: Local distance matrix has shape [6250, 50000].
+Run 4
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [4/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [3/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [5/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [0/8]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [7/8]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/8]: Calculate diagonal blocks in global distance matrix...
+Rank [2/8]: Calculate diagonal blocks in global distance matrix...
+Rank [6/8]: Calculate diagonal blocks in global distance matrix...
+Rank [1/8]: Calculate diagonal blocks in global distance matrix...
+Rank [4/8]: Calculate diagonal blocks in global distance matrix...
+Rank [3/8]: Calculate diagonal blocks in global distance matrix...
+Rank [5/8]: Calculate diagonal blocks in global distance matrix...
+Rank [7/8]: Calculate diagonal blocks in global distance matrix...
+Rank [0/8]: Start tile-wise ring communication...
+Rank [6/8]: Start tile-wise ring communication...
+Rank [3/8]: Start tile-wise ring communication...
+Rank [2/8]: Start tile-wise ring communication...
+Rank [5/8]: Start tile-wise ring communication...
+Rank [7/8]: Start tile-wise ring communication...
+Rank [4/8]: Start tile-wise ring communication...
+Rank [1/8]: Start tile-wise ring communication...
+Rank [7/8]: [DONE]
+Rank [6/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [6250, 50000].
+Process-averaged run time: 1.1198103771312162 s
+Rank [5/8]: Local distance matrix has shape [6250, 50000].
+Rank [2/8]: Local distance matrix has shape [6250, 50000].
+Rank [1/8]: Local distance matrix has shape [6250, 50000].
+Rank [6/8]: Local distance matrix has shape [6250, 50000].
+Rank [4/8]: Local distance matrix has shape [6250, 50000].
+Rank [7/8]: Local distance matrix has shape [6250, 50000].
+Rank [3/8]: Local distance matrix has shape [6250, 50000].
+Run 5
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/8]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [4/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [3/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [5/8]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/8]: Calculate diagonal blocks in global distance matrix...
+Rank [5/8]: Calculate diagonal blocks in global distance matrix...
+Rank [4/8]: Calculate diagonal blocks in global distance matrix...
+Rank [7/8]: Calculate diagonal blocks in global distance matrix...
+Rank [1/8]: Calculate diagonal blocks in global distance matrix...
+Rank [2/8]: Calculate diagonal blocks in global distance matrix...
+Rank [6/8]: Calculate diagonal blocks in global distance matrix...
+Rank [3/8]: Calculate diagonal blocks in global distance matrix...
+Rank [0/8]: Start tile-wise ring communication...
+Rank [2/8]: Start tile-wise ring communication...
+Rank [3/8]: Start tile-wise ring communication...
+Rank [7/8]: Start tile-wise ring communication...
+Rank [4/8]: Start tile-wise ring communication...
+Rank [1/8]: Start tile-wise ring communication...
+Rank [6/8]: Start tile-wise ring communication...
+Rank [5/8]: Start tile-wise ring communication...
+Rank [7/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [6/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [6250, 50000].
+Process-averaged run time: 1.0991524877026677 s
+Rank [1/8]: Local distance matrix has shape [6250, 50000].
+Rank [7/8]: Local distance matrix has shape [6250, 50000].
+Rank [6/8]: Local distance matrix has shape [6250, 50000].
+Rank [5/8]: Local distance matrix has shape [6250, 50000].
+Rank [2/8]: Local distance matrix has shape [6250, 50000].
+Rank [3/8]: Local distance matrix has shape [6250, 50000].
+Rank [4/8]: Local distance matrix has shape [6250, 50000].
+Run 6
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [3/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [5/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [4/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [0/8]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/8]: Calculate diagonal blocks in global distance matrix...
+Rank [5/8]: Calculate diagonal blocks in global distance matrix...
+Rank [3/8]: Calculate diagonal blocks in global distance matrix...
+Rank [4/8]: Calculate diagonal blocks in global distance matrix...
+Rank [6/8]: Calculate diagonal blocks in global distance matrix...
+Rank [2/8]: Calculate diagonal blocks in global distance matrix...
+Rank [7/8]: Calculate diagonal blocks in global distance matrix...
+Rank [1/8]: Calculate diagonal blocks in global distance matrix...
+Rank [0/8]: Start tile-wise ring communication...
+Rank [6/8]: Start tile-wise ring communication...
+Rank [3/8]: Start tile-wise ring communication...
+Rank [2/8]: Start tile-wise ring communication...
+Rank [7/8]: Start tile-wise ring communication...
+Rank [5/8]: Start tile-wise ring communication...
+Rank [4/8]: Start tile-wise ring communication...
+Rank [1/8]: Start tile-wise ring communication...
+Rank [7/8]: [DONE]
+Rank [6/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [6250, 50000].
+Process-averaged run time: 1.10608710127417 s
+Rank [1/8]: Local distance matrix has shape [6250, 50000].
+Rank [5/8]: Local distance matrix has shape [6250, 50000].
+Rank [4/8]: Local distance matrix has shape [6250, 50000].
+Rank [3/8]: Local distance matrix has shape [6250, 50000].
+Rank [6/8]: Local distance matrix has shape [6250, 50000].
+Rank [2/8]: Local distance matrix has shape [6250, 50000].
+Rank [7/8]: Local distance matrix has shape [6250, 50000].
+Run 7
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/8]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [5/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [4/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [3/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/8]: Calculate diagonal blocks in global distance matrix...
+Rank [3/8]: Calculate diagonal blocks in global distance matrix...
+Rank [1/8]: Calculate diagonal blocks in global distance matrix...
+Rank [4/8]: Calculate diagonal blocks in global distance matrix...
+Rank [6/8]: Calculate diagonal blocks in global distance matrix...
+Rank [5/8]: Calculate diagonal blocks in global distance matrix...
+Rank [7/8]: Calculate diagonal blocks in global distance matrix...
+Rank [2/8]: Calculate diagonal blocks in global distance matrix...
+Rank [0/8]: Start tile-wise ring communication...
+Rank [6/8]: Start tile-wise ring communication...
+Rank [2/8]: Start tile-wise ring communication...
+Rank [3/8]: Start tile-wise ring communication...
+Rank [5/8]: Start tile-wise ring communication...
+Rank [4/8]: Start tile-wise ring communication...
+Rank [7/8]: Start tile-wise ring communication...
+Rank [1/8]: Start tile-wise ring communication...
+Rank [7/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [6/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [6250, 50000].
+Process-averaged run time: 1.1259497278369963 s
+Rank [5/8]: Local distance matrix has shape [6250, 50000].
+Rank [4/8]: Local distance matrix has shape [6250, 50000].
+Rank [6/8]: Local distance matrix has shape [6250, 50000].
+Rank [1/8]: Local distance matrix has shape [6250, 50000].
+Rank [7/8]: Local distance matrix has shape [6250, 50000].
+Rank [2/8]: Local distance matrix has shape [6250, 50000].
+Rank [3/8]: Local distance matrix has shape [6250, 50000].
+Run 8
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/8]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [3/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [4/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [5/8]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/8]: Calculate diagonal blocks in global distance matrix...
+Rank [2/8]: Calculate diagonal blocks in global distance matrix...
+Rank [5/8]: Calculate diagonal blocks in global distance matrix...
+Rank [3/8]: Calculate diagonal blocks in global distance matrix...
+Rank [6/8]: Calculate diagonal blocks in global distance matrix...
+Rank [7/8]: Calculate diagonal blocks in global distance matrix...
+Rank [4/8]: Calculate diagonal blocks in global distance matrix...
+Rank [1/8]: Calculate diagonal blocks in global distance matrix...
+Rank [0/8]: Start tile-wise ring communication...
+Rank [6/8]: Start tile-wise ring communication...
+Rank [3/8]: Start tile-wise ring communication...
+Rank [2/8]: Start tile-wise ring communication...
+Rank [4/8]: Start tile-wise ring communication...
+Rank [5/8]: Start tile-wise ring communication...
+Rank [7/8]: Start tile-wise ring communication...
+Rank [1/8]: Start tile-wise ring communication...
+Rank [7/8]: [DONE]
+Rank [6/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [6250, 50000].
+Process-averaged run time: 1.1099530989304185 s
+Rank [1/8]: Local distance matrix has shape [6250, 50000].
+Rank [4/8]: Local distance matrix has shape [6250, 50000].
+Rank [7/8]: Local distance matrix has shape [6250, 50000].
+Rank [2/8]: Local distance matrix has shape [6250, 50000].
+Rank [5/8]: Local distance matrix has shape [6250, 50000].
+Rank [3/8]: Local distance matrix has shape [6250, 50000].
+Rank [6/8]: Local distance matrix has shape [6250, 50000].
+Run 9
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/8]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [3/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [5/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [4/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/8]: Calculate diagonal blocks in global distance matrix...
+Rank [2/8]: Calculate diagonal blocks in global distance matrix...
+Rank [1/8]: Calculate diagonal blocks in global distance matrix...
+Rank [3/8]: Calculate diagonal blocks in global distance matrix...
+Rank [7/8]: Calculate diagonal blocks in global distance matrix...
+Rank [6/8]: Calculate diagonal blocks in global distance matrix...
+Rank [5/8]: Calculate diagonal blocks in global distance matrix...
+Rank [4/8]: Calculate diagonal blocks in global distance matrix...
+Rank [0/8]: Start tile-wise ring communication...
+Rank [6/8]: Start tile-wise ring communication...
+Rank [3/8]: Start tile-wise ring communication...
+Rank [7/8]: Start tile-wise ring communication...
+Rank [2/8]: Start tile-wise ring communication...
+Rank [4/8]: Start tile-wise ring communication...
+Rank [5/8]: Start tile-wise ring communication...
+Rank [1/8]: Start tile-wise ring communication...
+Rank [7/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [6/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [6250, 50000].
+Process-averaged run time: 1.103599097696133 s
+Rank [2/8]: Local distance matrix has shape [6250, 50000].
+Rank [1/8]: Local distance matrix has shape [6250, 50000].
+Rank [3/8]: Local distance matrix has shape [6250, 50000].
+Rank [7/8]: Local distance matrix has shape [6250, 50000].
+Rank [6/8]: Local distance matrix has shape [6250, 50000].
+Rank [5/8]: Local distance matrix has shape [6250, 50000].
+Rank [4/8]: Local distance matrix has shape [6250, 50000].
+Run 10
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [3/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [4/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [0/8]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [6/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [5/8]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/8]: Calculate diagonal blocks in global distance matrix...
+Rank [2/8]: Calculate diagonal blocks in global distance matrix...
+Rank [3/8]: Calculate diagonal blocks in global distance matrix...
+Rank [6/8]: Calculate diagonal blocks in global distance matrix...
+Rank [5/8]: Calculate diagonal blocks in global distance matrix...
+Rank [1/8]: Calculate diagonal blocks in global distance matrix...
+Rank [7/8]: Calculate diagonal blocks in global distance matrix...
+Rank [4/8]: Calculate diagonal blocks in global distance matrix...
+Rank [0/8]: Start tile-wise ring communication...
+Rank [3/8]: Start tile-wise ring communication...
+Rank [6/8]: Start tile-wise ring communication...
+Rank [2/8]: Start tile-wise ring communication...
+Rank [4/8]: Start tile-wise ring communication...
+Rank [7/8]: Start tile-wise ring communication...
+Rank [5/8]: Start tile-wise ring communication...
+Rank [1/8]: Start tile-wise ring communication...
+Rank [7/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [6/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [6250, 50000].
+Process-averaged run time: 1.1231002451386303 s
+Rank [5/8]: Local distance matrix has shape [6250, 50000].
+Rank [1/8]: Local distance matrix has shape [6250, 50000].
+Rank [2/8]: Local distance matrix has shape [6250, 50000].
+Rank [6/8]: Local distance matrix has shape [6250, 50000].
+Rank [7/8]: Local distance matrix has shape [6250, 50000].
+Rank [3/8]: Local distance matrix has shape [6250, 50000].
+Rank [4/8]: Local distance matrix has shape [6250, 50000].
+Run 11
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/8]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [2/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [3/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [4/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [5/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/8]: Calculate diagonal blocks in global distance matrix...
+Rank [3/8]: Calculate diagonal blocks in global distance matrix...
+Rank [1/8]: Calculate diagonal blocks in global distance matrix...
+Rank [6/8]: Calculate diagonal blocks in global distance matrix...
+Rank [7/8]: Calculate diagonal blocks in global distance matrix...
+Rank [2/8]: Calculate diagonal blocks in global distance matrix...
+Rank [5/8]: Calculate diagonal blocks in global distance matrix...
+Rank [4/8]: Calculate diagonal blocks in global distance matrix...
+Rank [0/8]: Start tile-wise ring communication...
+Rank [6/8]: Start tile-wise ring communication...
+Rank [3/8]: Start tile-wise ring communication...
+Rank [7/8]: Start tile-wise ring communication...
+Rank [2/8]: Start tile-wise ring communication...
+Rank [5/8]: Start tile-wise ring communication...
+Rank [1/8]: Start tile-wise ring communication...
+Rank [4/8]: Start tile-wise ring communication...
+Rank [7/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [6/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [6250, 50000].
+Process-averaged run time: 1.1025142738362774 s
+Rank [1/8]: Local distance matrix has shape [6250, 50000].
+Rank [7/8]: Local distance matrix has shape [6250, 50000].
+Rank [3/8]: Local distance matrix has shape [6250, 50000].
+Rank [2/8]: Local distance matrix has shape [6250, 50000].
+Rank [5/8]: Local distance matrix has shape [6250, 50000].
+Rank [4/8]: Local distance matrix has shape [6250, 50000].
+Rank [6/8]: Local distance matrix has shape [6250, 50000].
+
+============================= JOB FEEDBACK =============================
+
+NodeName=uc2n[135,137-143]
+Job ID: 24674018
+Cluster: uc2
+User/Group: ku4408/scc
+State: COMPLETED (exit code 0)
+Nodes: 8
+Cores per node: 80
+CPU Utilized: 00:04:51
+CPU Efficiency: 0.44% of 18:18:40 core-walltime
+Job Wall-clock time: 00:01:43
+Memory Utilized: 1.93 GB
+Memory Efficiency: 0.27% of 703.12 GB
diff --git a/2_psrs_cdist/solutions/A1/strong_scaling/strong_scaling.pdf b/2_psrs_cdist/solutions/A1/strong_scaling/strong_scaling.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..75a70873a5d551d04fb6382ac7c9040d84fb304f
GIT binary patch
literal 17643
zcmdUX2|Sg-*MBO~wN#cQOBW%+-LEBNk1TO5S<5clC2m4R5|MphLXn-y7Fj}eEtV3I
zQjsl6BHH-Rb5WAtTkrq#|9?L3duy8Knf1(>Gc#w-_dM}xDkuq~M6fX4{C;>|1q=m8
z!kx_RV3Lw>gpP-^6&xXNMly47vVkKs&1|e(;b>5R0GF1ASrILvf?}&GC^->Ha16Ns
zLjQ=los|U%F1qxp=uSGMO)?`{!LdsTnr0-Dl?xG$gI-|>?Gt8}wnQ5^emU30$wJ$T
z1UCZRDkuR`tlUX(go+~|L2fygUyfDaCacIn{W$=LZisppD?r^6`3QoQtJ5hL3osst
zegYh!Ze?j}Cg<c1dPIUhVt6bZgGa(mU<i3IC{WWCF1}P!mPm90Zz!nPYDPfWE1Xla
zBHEBnz){P+DcL%JvBD8b4uAm)Ru)c{RuIXqBo`|)N0@t3qoJxRlj!#9_FEk&cA>n=
zSxGtNY|R)QiktU+-h@A+@hS3PT=0^%J#CXdn_tCvD<m&CA#?7^`y)5J5}q{Yc@L!A
zN;&9}?&XC_(l|15oQm*a>W32REd9U$a}NK!7iO2F30<er_bQ=-jUzP)kJ`QcU(Y&!
z-JW|qx--J}6rZ96O;;30r|FH)$9mISH$6p<4ZRrt;&OH;wb((H$w_UGC;Ynd%G8Fm
z(x$;w3nMXKweKFy+9qyPKQ18Yo~3*#<lv3w6w=;$cBM0Xt)G+YgI*_mEm6ce3-Z(n
zz<18Q(%#lw;UMJ9>C^wAp8vaTS0Ke@nnYgSBCesi2)F3HVyC=tk`inh-n@!>MyEbk
zB-QpdP7!`6Z0p*8>N9Ent%JMXganLb(;e!3ivm=LNbm(-wTVN$?Y?0JeHGi;`NhMx
zRAV?JpESW~vvwu#RgE8gJu$+~>Zlk=!8JiYRQHmVW$#pv>x{PP(KKF%nTXGyWH%mX
zIA9afp6b4x>th@BJ-=?Q-P79VLH0$E%~;2DJ8mf7J=3??zXqd}U)VxYHbcLiQ7O3?
zA^q4(Nyh*4y_-))Q@_Y3ZYFF=V%eO;SU1blDKNFU0VCsmw({$lS%Y)#H(q+*(LYyy
zAcbZ-YnJ?!^wZnL44f@9v^%Y6jMCo~vXpf(7#FmuD%@oFBFk()NF^AI49a6=wF;un
ziReDtB_<_tR>sbT$LCR4u8}=FHv4*5Hinra#(C==VLh&N#c^aEcVF)-My{)07IdpD
z4o{tQRH>q2<^=_Kh4oTF0ZoE&tz#kHWvG7t*DS8VBuj-|A8eBolTMV14rPuEy!1YI
z<~AW~e3wTnGd|c2^+w6L2>XhwY&OzG`_tL7%9~egPxg6SX1dZ8S+%o-0u>jKbKt^m
zu^_q4C!g|eFVqmA<Ox68h~BL(TjZjPkbD<DRjodYa=wl`hHLUZ7A8e{)NFG5QEzPZ
zsdt>0ni}5>e~FR!@glo<kJ2e_$Ew*B-%Ydpn`;~bE^InwtCVcO_|>dLdN1uMtQ?p4
zp6zz%tT09^mb<=+@f{jxLn}wsu_^9KQp4WcE~LK|%VoTmP%!6D$&ay-G;au)ZAw$J
ze76IAkMDA<yyK(duY7NxB@ngP*bi8kVuoK=^_5SJ_Zi3QR`?EO7c4Xg)wx-lT<LAQ
zF<@iU^NOSvrHdM<`xx^s>!izzQGBs|xhGvq(dIHt;q$%o;dxuy50_+r8RwpKsup_F
zGAwiF#r89jhuH}mURPz7dPWLyrC)lI%ELR^5&qI8R*IWr;!|WQr~Lj-Zn+BOWZ^?1
zhVd_m;*U8aBG13X`FS$kPzc&L#(iqvM-BdHA0ztP&DR?$)9K1RcYzGNnMryZJ$2s+
z7*VR5GmKHk`s}`^gC}?m>s0pG9^~KGB9TB!zt_#17Zrb<uJlm8IorPIBPFNb^y?e&
zxBBQhP_+)T-i&y}N8D^HGvi~zc6-~_*28=KMS6xgl~pLLdHR$zl<uJq3nmHk;0R&i
z4RD1H3UuOA5w7=~)Lg+P0#xnV@j7W%=~#UuL&FkQb2ZMo>8>!<2@FOpe*9~~xU`%f
z_x;1^$Wxh7mRtzt$I7Z9Pc0}1<L4}ehp~*`^7{`}A^G<nV2R?e%~5#ZZd*{=P;!F9
zIvy1+Ho~28W#@*odp=%B+gKm+TA*Af&F^8CJ`aY@FpO0#Z*;@PF99B9?GMj2&_;4t
zE2?^3PB>f=oIqv&KrN`-jH;FLp>jihsMx!sUd1-{mN!OC?<W#U!nhvOWh^!}KOKMb
zyxC~Glw<HQ!K{<%!KbV0Xhi(q<(>C@oYak<tz|wRvq_*?3rAAe6`4Qhu}25@^5Pjj
z6ju`bR>hIg;h2aZ5ACCKE>B$SB>NVbb*$v^69b$k1K}@5#Wfx#TGHK4iMv};+t}^8
zox6MZso^`Zv`2?da@`K)`Ch_u<wsR9Z|{&<Z=~bh^f$J>Y@G|d2Gx|=H(M2=tfQYZ
z8+4tB;jnfpb#0mSq7B|fMaNFt%R1>M)uc6Y{63%jd}49Mi%^4^Nn{}Vv0eLT>*FjA
zoXLD)!qAp`Ov+?3KV|xTUS&=$Q|Y@xd+MxhR+a46BVSdsxb`gaS{6iz_J)v1@2KwI
zb2@fV&AcH?ab`!f1+Uv^(IJ!WfUt{&w<7K<&7RF}j1BS$Q7(!4o-uy(?I*q4DvHk1
z2FFqmAJx8m{)YZ$6QQ`NSZ~)!{vA$8eEao?xEKdgmOrOdO<TXmOFHS@M73mBABM@@
zv~h_KGsDy?iylO3x>>!gyS44}+xZKkk6M)3v?&#{&*tVH%)G%`W0Jp6k-uL_B?wc4
zy~D*D%89bwZ?%wZoOm(j$nB|Y1KJNSV(kxF&~gnr3<O=Q$V>3cxo&Vp%C6|4Ek52&
zBgzIlD8$!ZCDr!Kz}&^b5wHHp*m+3L`bX-I&W}^FZFiVbXP$iNadJ$TWLwzu;Wou#
zND)~+Ntaa;^05mY#V9cxZbeTapVv^L5jfOW3a)czi{imK{g?AvD&*dd^D@+%7Ih*R
zwkuS5u{Uq&S?rOgI@n4@<&x`*vJCt{73X7rJtm4zfQ7R6b@)}gZWhI)1HvpeyF@d*
znLDrga#kE~`N*>PC-G=VuUMl;pfRX_AWBokr5;IJ`HlJP#ztMwumj2L{fozGkO}N3
zYTr;57%9HpC7n!tV7geOohOyoJMOBK5N)9`_x94a)oD!Ld|KHzcB?&YJug148PWSB
z>TdkK=zf8u#Ad%U0drPHm!#_NeEgE@ezT{_P<uzh8%OLmO>FMBv-%|Z2OC7{jbr+B
z^f(EH8P^1l)ST=Or?i7f?U3=sNVV)O@sxMcayoo-=e7!!d6#CI{z_r7O@bK8r(Fi!
z<|nPQY<cQ=wB5XrcCCFuv`=V@>o-c_+k|PPsyjojNH9%Ny?3grIr`eqYms8eLhsc&
z{93K>E$dwvEavb0!VwM-c~P|QMwqH(HoUXx9r3tl9yIaeIWJC`SFTj~vfH;hxdYWg
z1;w3X^QBqu6>o)j`{uGAJi@GnV6qwCJS{dZA$`Q!93g+RzG6qR_Q50b97Ct&)!L#Y
z9x?LUoVq8Ve|uoJD!*Lmv+|ZJQ^{S2lV@bz;CfuWA?ZPp8WLI}<;DscF^<$8HGa=3
zipO~CY}60V>apAkx7|{tsa#t04B_NZm*}46DOiSP+iP=t3vmmY=qQFHpt%e~>iP0s
zWTIw6RGwECyg_aia!h-c#NGb1<jGL-;Z!dzf~F`Mi-~v=eNv=6u@#YrD5N8>D=JO%
z%THh48klUt7<Af|t7%c1>MD9yfBRq%S1-qV)4hXPw_8%H$`>xj8#$%xP`Z<1Y(mS=
z?X^A>$8M-`^{}N>_h8k&yB)?j(HfHznfqQotU6J~I{pFiUDEsN!9@xQC-0PX<_FS&
z*UB>%_xJfx)DkCd2J`a+cFuIeyOJL%TXzcJ7oFto_>~pwBKHxF8KSK&8IKcGt)50B
zII0OuM-K9IB`T_E?ca($;or}Lf2J3RX`HzCHvfEiMnV04&!(`#jWHcOI&`OBbH0)4
z?-{X5d0%iYg*ESqU~HCd;8m$d1qmFax`Z&%+zw&gu?0lCRP`A_JFY8^LlRl1Gg&Kq
z;%m=3wq9(ujdVUT&~RqR_)(Fj&p_gxR*7sK8I}cirFOfL#?6)$Wd-sPAsAgIUut8m
zPoBiHw^iTsUJu*<{F#S&g6g+p=K@};q2EePg>9Q+8~Cx%9{(le`>o^0bxf+(;U09G
zTFX6**x$K_3sWU-Lea|4xx!4~=Vyy2KB;MvI$wIT5!yB+1cdWW=aBA)D|_TTAVnq0
zzw}hp2sD@;xcsQ*orm=07^fpQ)O{}P3yd%qIi_9OA9sEB@X3%+REfcd2TgVvf!hvp
zp!vVfYrklCd|N-C){*)U!?F6Jtm~ugv!q@1A`~qf^!D;ZUf!J0tuUsDfM@!|He4|n
zdG+lp<*m9GC#!nwm^Udj8F$gX-_rH?CFWcAbY{|<SA!SZM>z%_ZHuclA<<2tFEJ<8
z8yQ#~(H}nDEcv`~;!aUYlAiMLW|cdx&qn18D<T)lFO~f`yTRa`@!fUU23<1OvJLky
zo9IuW^D?C%3*7~#Vjy(3kvEi;(Tbhrc)tEA2KhdPp@z8Nw2F-%F4?}~%$+u)=%t_C
zy4BZO?x8cC)W#it0{f%xlziMReHt@2E;OI%@8Bj!mv$*HaAKIGte3UR>cb9|1ix#(
zLl0Ccd2QKlZ90%QX=KS`(xAOV<8g2YG4>MfdD77=*|^*E!9R3myGmRYpZdMN`0-&!
zTjPnF1;->~;*0q&FjmjD7h#GnKO#)+8Nzw_Z?Uf$`GNEwrtJv5=Mi6a>Z)Z@p{_~F
zhfa4oM%&Dp;kTD#O%5BFWJi8aQhzF6^3wGj{$>Bd4;rT{MuO|`5W1PK;UNn152vGg
zm@LC4F6zx6sxI(vxNzq~ua4ce;l-ZdMBJH^73>+i5%=mmUpb#{6Z}r7_=w5zbk2I?
z5!IW{?ZZ^M$sqbj5tW?1dimit+Wru#i(8YcT721WeREjcs~l6|igWHSSO|MBE9|B7
zI={XU*PWe|i0p84uIn?EblYDs-T&OJV)O@PC=cfGIs<}i3u^`>iv9ai!r)PQNX-Hy
z1%py%ENs}C3MSxhEK{8mSS}~RqD<RxpQ4!dJXYV9UQLnyC{@7L$5dLh=CVvzSQF*D
z?y+t!r@e{Yus3>V(>fGE#*sA?p~e0|Oce<#k(*G|;H;OtXFVb%g^Q_wxOi)aTb8S`
zDy0&2OX7X30OeH|u1_il?e&ETu3z_5Uo7D(XX-_}Y`NTltEzR-Y7q#EXS)?qcg2c3
zZM(m}x>9DBN@u3gSa3+u(?s>JY!M&fSyK7?(1EH>YF${yQBFyOr4;LYm}R7bx@pQJ
zJYSjRr66|_(*sd+&ML>@o$je;%w?S{-DPjIu!N5&HF1Vh*p~@@es%JNTGuuG=dyE>
z^d!zL7jC#6Jb9NRpg$t#IYrh0{WI4Crr-Ny2Tyk7AILgei<OaBuo}Pen$wWt9FOl|
zr<V<t#fW}}%DNZdT<*IK5PtL%Xm->~8E8FmjZfVzB8=qy{Nqcz@gUA?p#0oP$#poq
z+B~&ll@Uk&ebN=fRUr<`&q@1*o|m0>P0`4(MwKt((O-lovgC+3WMzFpw*BUadgphO
zEdjOHqhHscdg5BkrXH5TT#;GQY4HIhW0#F6x*?_f?aY-A0=Fg#R?FP3l_qtcRNLzH
z(x4o5=%d|^Z@ObH60bdX*tdL4{XTwZqSD>YC0*lr(4M_n4LOTUhjL<GNpB4$9n#df
z`t?)vV}+7@x`?~(w@Z53La#8zM7Z&>n=poE-+0U?z5j0C@yde~wzNto)?vdsiY^-U
zFG`YTIUP7g-bwlGq&6Kdc1oeSfbFdb*~j1LBY7oFdE^+nQJoX1ayl?+x}hMA$aU2W
zlQ<@*6QC4e!Z+6;aXX()xXZWVYvy96rjOJas?;h0mTVdm3hKG-AIyxdc%0L@^ICM!
z$L(X6o8yF(WxpC_l=HCVb($xSW5yQGz5n>0nE!>9SZNR~5uBCLcPzI-@P`M(1790{
zR=4EnjU6c}O~LdcGeZ_GG=&n)4*oGrGLy7JPpM|-XZFv>6cv9Q-=K3<ncYUY4>eOm
zwf}B4x4(|*pWMN$r*dQeWf_JafK(LuGgmLL=+}PjlWyM}#MJw(YzK}potr{ss5ovf
zU+G|mD_0!c<&CO;(>;m#lCjtA^G5b88ueo}pB&2QyyzXe)tId_N}cH*gi!k(ndF@7
z^$f|@uV_h(=d!8GTiiVslh^66vo}{&T!8<KFB^B36Pi_z%iMH0FIu?Ez924?{niPj
z-gX_jM|Y>HzV9BW*Q*U0#^leB-5GzHQTg(;a;kyI-S&L3oQh}N8bPIaOVt78dF_3P
zrn%u+o<n?0j)$y??mt@34W4nke6=^{#tZpoYhw6CQBgK+<p*7v-Y@IR?s`)Yy^hVU
z!?|^oZxrqy)DuNb@)=S24VH2~a`bD;B+2vH!eU=-=@VbRi)u23_5Jjtb#8;CGc&Q+
z&t@O|DSe;RI*y`cPhCo;<B8E_{u=Aek|I2>OL|pMXj_;mVR4E^i0v@VRX<;z1Lnk|
zdFrrz1G-<Y`ZR{#w7Sj^?<%+JVQZ99?z=hd^T)CsW3*uM^t^%mxtNg~`3-{$RKuk;
z$M5vzNgZ@@6cbu!nCs}U80<eBg}6u+q5_IqVNRNo<wUC*XWOUidOMT9IemiPhdDTF
z>54hskH&iBs5V6yYgd!fbJLG}8GHC4wmo2y!H%JtS7<Mvji`tJEuJ0P{ZYH#U?imL
zKTkX~?>y63OWTvXPioja>T+AC*ImIFN6!S^mL{R&Kbms(I!y=2cZ*=ogt7^lZBQ-f
ze^qbE-=1Th<;ie!w#YK^eQ4jE<WxRKm1BW}%CD-iK9xtGTO2)5$Mfj}RWu8Rsq|L*
zHOW_ES?mTGfd|s+zF>#}=HVM}SO>pM>*GTi?r=<1P~LcShrxJq`PKJR##IKoW~_Ds
zUNUvhLIbW7dP6gBTncUM$9MM@n;q)4aOKI~TT&QwnQt$1-u)?aTQis5`U>9bUyG{`
zJC@hw*hdBk8}aJ<cb9dVr>ZnPwhx@mqMc>7*UiH;k3D?!*yjPkXb%qs4Hcipf_I(G
z*|0f8cb?bob*5+?MH%-Gw+sv(ql?@iXQIkOX#s!U-$LMO41dtl#s2W&+Y4KVomqG5
zpO1O#%qn9WpUZv|I>}+#^m^-bzmCF*`iFIZ!53;G&(|}_gO)j2H|eg^^15mm`X5fP
zFj;zeE+BhJ7kH@6J3l?cnmqgFbf9H%;G$T2um;tqHouHF1>Mhrrj3-PK9=Pqd`c_W
zmLRyVUmks59bqxq(s+g=V!(qS4CgS$m7H|UiFz2>otcal(GB#v*;1#|q{d)utnX&3
z(zs>UHKB_!oGpE?IWEt37mubP&Mg?g(Q4}qW*rp_jr<24OkGQsxS5N#@)OM4l$T*6
z*N0xar}Ezx?H#R^##{OLnzq`8hW5Mn)=r181|sb9#rD^53;w`f{TQbBr9QTPL}p%6
z%&obS1ONVTO~(Ub<Jk3Mq5?E}@Ak39(r6rtdiN%wEPLo8<^BVv@){E#)6t%htp2Ct
z9jnXf2`UGLDhituj@-PzlT@Uxs50t;e9PzdXg>dqdGfx=dqO*I^C>YW$={S4>(z2s
z-xsGhuEsS)`@^sZZ9FD_UDvcyy7rm7(~lR`&qua-gsQD`Lq@IE0+%p1GLVLlGjp|C
zdQ#L=k(WKPU%|@GOz)Jo8PQc(&dI?Nk^vW2lKcrX7dR4zP%~S8Mw0~}z*Y)Yt`;t~
z&Lk%nIBFSK(>`TRBEx!6M;I7F!^{zYWB;*z(1_d;Dkh9ViX!2{SkMqk92m(^C{Y{)
zk3j^gn~_{>$<Uby5(ym%;Lpz|696Fs3N-<u2T(II+y+A^5&;MX;eEew#+4)(Lc!MB
z+6tg`5IkoD#{gM(bvCmA$e}2Baxil}0T%^JO9L(jatLsQHe4LMngc`*=yiZ8U_92~
z1uTAB5Q)PPj&KAKhH!!-oB`E9*jzv}a0I{;Pk{jd91d~R){=Ar5J$!$SA&VG#r4<B
ztYRy9_<!M{gPDyh7{wB+{!4Dc5wflp5K1YI24q9sKrg~*=+-4~=B#XGYjc7GO||~Y
zBfvlbI%{hoOSEyY0&GWUldK%|pa*p`ck-~{C@dDadQ2<(i4hzR820}l!2WOBz+lDT
zSR5XXL!;ngz^H*2$HTxRqTynqXgCIm2brSefJ0-*1u$YrI2Ozj6l3r>sKS2=u+@23
zFi<E!ovZ{LXajVHgo}bZpeP;<7sX&fCg6w|h8)DmREfec5HTQ#0(r+{MB(D%I8aU$
zE`|ejF@P{A21`Q}SZ?t^B*lRF8EO-SLSn!J3S@u|&^Q=08i;BrhN!@S&Y^$@4U2+h
zg%bzmL5v5|jm0dbiGs!;YbF%%pe~33i?E;vkbxHyC0BqdVL@dapbQHtlk;$BAVpX>
z9+=wj;28%fBR2*bfnq?w(!4<|1VoTo3p&LE;>c`Sp$>}4A_R>O3u>ZJcu{f{Fs2pp
zfstnfa1^Re4j6Ir*nT#(^Z*eEHGvcdltIIXr~*{3NCG4Q<cR|k55*9>RtQ;Y0pRj@
zpm{<3gCyZ+`%o!zB9yVjT$m`-3J4H!P(C?*xdAXaQ0<j}AZZ{A0wfG;0tzXH18sqh
zmnCB*UXhHI80vpTG9an=EkJot8VU`ABxEI!g#+Y?qQuF9013m|045wr3z%R?GFAoH
zie#+BO9HVZ7|`?T01>y64++3ZfP@1G5Dbcc0VAug%^G0C2RjPbX$f#LIzirZ!RjEV
znZwb`UJjOUuuREmR&X$<<R@!55EAeN1A8%g`3Q1^qnDjRh;a0>`-Kx6?0DoF&S2#)
zJ-NUE>_JY0w$G(jmbQPe>Xy>U#{!rc@*8kHS=zwCz7A#`O8e!e0<Lp^m|DdD;fjX~
zuB59)QY)Ovyt89o?EG9{8q?<m#{glLeyOp*PY$M*$(MLa26q-^I%}&DG_p_A+GU7x
z92n(&^aI=KDYAQJ+sRqx(kq)Hwf06_QiyvHWApa-S!r4)cJn`#Be1IfAn{*pmY5=}
zF0fCPY+&$@9fl1n#ggq)(g86KC>bC8W}7;?dqYhy-8$+k*j@hU8iNGK?O&0w$YnFL
zw4ccK+ONGSpVIeL#J}`@&d)n`nZ0+XqFro-nt?WFI5z5{_t=zkVe<3=hjwf0SB7bE
zfp4QnI+)Ghr=9J6sAOWm-#C8JTB2-JOPpm3UOo3}o>0aM5qJ6?z7(U+I}TQLT;uG*
ziEI@m9ldTPfVIf|=Kc80_h9rZJ2evrLf)A@lE)NuM@P3D3gotkV;LEg-8y!-?u21L
z%xT3<vhSx$?w{HE{`pJ0mZB{=@ofhLtP9?e2qHw5OI(~S)7evRuRpVwYo9OJ&pV_1
z%pc>z|NcbNa97Kl@P(Hln^nVZp5XfQ>K3tE#I#CVLebn(!zJG~NLOP{lj%u${axDa
zvkr>#0<IBH$B);y(b&g2>~uf;m6G%N3qDnPz24{98Xs;*_IAweG3+7jS!deUxjLXB
zrRQ(F$7!h$8_0m)_u38J@2mJ5uU-1Q*h9F_L@mQxdWYBCf@dQFH7_2}KOc#^&gW(0
za`d~<5x+xqH(seQ`NX|G=62x$Ta(p6dd!}xIK8&?rYPDLMZA>c4)inL?GngJs#DLb
z`vR-EUKjHa9x25>>0y|8EV%i;cIm*_1qR2wudd(nX&k@xF>1J-8CQAB7%&o>Uu!kU
z9Mz~#C8braiaHlQ?vTo#TD9-0uY<H*fI{3pn^%GN?(OTKpD3TSd85m9!sJDkqeZsT
z;pQhbbn5Dn)L#^~+LU77?_BV1zgyk&ZJP#{&W?2$wxYp6y5q7<@1GD-+!66eaEW-}
z;U|sM-Su3iCM&BH|Llu*MP!0T_!L*`6Gn~%_XHlE0_*1IhL&*aSK4ZgHJ?kiG3L&_
z;^dAFqA~N2aPH0@B&LM2EV{+C>u>phpTBHyZ2Pl1weF)kZA6|^TG%Evh_@1hO&AM9
zPTk1T)<1+yrM*Zht&fJg>ZH%KhN6c@#Ul7qf<AnHF61WeMgNWlAHu~vGB4<P0q>hO
z;qhVPZWq}_lNa-xij||haJH>GyS2>UC%wIR{H_~`^Y%UlSGh!45k{^@V}hC66E*XT
zuU{i*Ip;`cjpkqVetX<lcpZMN)?!!qg#`-c-vmrb^FXKq7wtQ7Kg87DX&z@zr95Hl
z4+A$hB?@S&d6i=JcI@yuM2m{PHOPD8+D)IDAi0mkH}8Dw;Cb={c$??80MbMzK7vt#
zBMEcROw=N@RuM&&G)Bsl+QKLPtqyk4mCb5GPnkb3mpgPE6Dp7*c;>46PNB&byb*t-
z)DZkNfe_f5aW(q>LC5>iFbV6<%|Z<2uPhnW#U50u5Q15w(c_$%ZrtbcC0njbj9RL<
z<?|%;J~{F{VV>Q&?0BE0Mocq_^Wl6_Nt>@YbH&_XAbyK_NZ;th9HXruf>6m}nCx|m
zA?C%dkzEXNHr}iQmWyAuEL<|=UWbFLZ)htRGtg!J&Ow4Iu~CNl`3RHowZ~Ie+C2!m
zFRH#d*~=4h1gip3O~`Zi!^_{QnlPO7Yb`r=(?~M&tca0ef=57QPzbLyK;GVSm70)5
z3dJ}MP1B&01r@R%<#wjd%f<R_3D=lPrKvk7zA%V5f3K&2?#^o|v~0oNVF9@&q84i+
zgX|8okqGAObj_;<bpu-O-_4;Z?VN%$Xe`)gww6%JIVv}79GBUkFcGbDBRte*FJle!
z3mbEG(~-_1ikqxD3bAJ<ngep$vV*G>8l;b?f87_WCal*))Mh>YJvzASEQwMu2Rj`-
z-Jx6DL8Bk&!SGr9yYl=<w*!~I1hIQM@a-N<BbuSjR8`zGHE!_GtLFx%cHHDBztcwY
z*%Q<4tXA^Qt7Z1c0_nNoqf=Eb)M+wr)n%tD>2E3(HZ-86lS2eN(}W6kCsrw8?wM`h
z(LGn+vT64Zm#Xd9EPL(}eeNr_%@sl}JGe^S-Q2DAp;5fMp0Up1LvnX&5e6g3b)m1q
z-|n4^TXFR#=hCYOF{5dY>&)qD73aURD%7=9E^h+c?g;Z&8LnedRax>0@*|5q_v!t_
zH*dWa85J9^PqPhmagUJHd<2)P=npopwBAYs&&bx^#rP3!ZLN7iwurf$NdQ}W>W(R?
zRC@-4!n+^Eg{~*KR&UwkcV{yumBQkj3c5UdIEIT+gZ30Vy<vLY9eeZoH-?>(A^pT}
zO4e3<7wN(*<@$G>4}ibhShPzh8joj08?<ejq-&)!H0{Y@(+aw<Ke~G>DNU~-_u;GJ
z;chokdEx0-+2=lfJ)sw8>es0{tM7i?PdYv)!R>vdoabxx1KbZeVd>}t!uxNZX&5K^
zd_DMpF!iH#Zl2@TwnyKMlh)zjI-mm3>;Jka6oL0lEu4DuqK@oV7_9FIUlcgDLbjvN
znvT%Fzeq__>Y=oUGKTLMv;LOD7X6xGGIw9Q2)FiAZgb}M?(5J=ihXk0HbO;Lpg*0#
zW0Bs5FFf0$(;|6}#^M{#ItUQZx&KgYAVb36Ha`ryOcQ8TSPBOJoqQ{(U+i{05f(L}
zEe|(##yZ2eQ|`r3Z78>)<NuR6Kymw{TX22&YqyF&^-a^wUVNXWwW8}@*3fO|CEnqG
zA!W?5H9>%gsh6ipp%wBsIq5SKi=;6=q#2q#aoy}`_4wwH*-XNx58I+sCi36-zoXbF
zx7&Ih($>*4F+j2ZD{VNin^B^upSKI#k;#}k()KAgH>euyW{h{4)sUux0tkc0s?}Y*
z&ok<WjP}g&7RtR0n@oI2Xq?+w%+xM?>rnfSGGW+eEk-H*1Ozj8SNqJUM7Hzct3F4m
z%b3`zd{s91+Rbm_%)r1kM`W5R&C!HS7xq4LwLM84=)xE1onDnPz<(n>bw<h{;lV(Z
zYDNc#y2dC)<NZdJYbGjRt|i1DR)~yycdk%w#=kaFQu+|8_51`r&@qjYZ-g_0_hAE*
zL_zb$D(l3jx}&CboR<WGkJ1!pb2>~LF~+2G+!$o-MZSE1)HY9jT|n72Y;JYo{o6s@
zt-63ottgXv=l8LXPE&WDSYX?E|CI%E<oQEcg$#7IVzmLaDaXVbk9}ekr>_mLNKO=@
zwK&0-<rx}S+mTY7Vb;{lX*m07o6gwTOY2O;I$Adh+-?3k5sG1|^qa}1-L%l!<~<`P
z(PMtDi<9S*iHsIfemXwo7gD7}>^CJd@8M!C1*YBDP(=f29Z!PZw$@jd@|zxT^<NX3
zx}YNH&!rg@zdJ(Midc63Ii9YlC`LOYv>@m@GVF7S12gCGG$Za)kq3Pn1QGp%6&x)Y
z-IXuXf~vjKDNtM28O=I+EC!gK{yG{Q;isby{5<jbhhDgH#P=GyoL;6)CrO0p1BN<o
z4=(xX+@2tPDT(Q05FI*g<=3#aHIHGE;IEurrN{O3Ljw9%8O~;N3n8O!D1GQ;>xF~$
z8Cf*VUrGyay=~zL<J4#7NmZytH#Us!j%|BiU~S|TYg-%Uelcl(w{WvQC4udQ*yQE8
zR1e`Is(`x*v6o>SdVxE=WItXz6mstKE**V%=Onv|b8w($#CbQLK4n_*GeT}!duA~E
zP>iE}Y>NG|KD?3f;*AxgnSuF^!fOl9f<<d21jmdF4J;l@vrR@f38BY4^=C+*x|@Az
zupiRCk36Miz*>bwHf_1lC+?q?prz+tCLcxj2-VE>i9V&Cb?f1;l1w#Wyo?;tEd%s)
zsouvquLmdLUv3;uJLt%~h5k)({jqW%=EE%Zk#iI|YNI}5i?=@BuI}mC!!db&9Zv&b
z5&mO4LjTL<QdmpHWs@v*W&8e_i`H2K7IFJ84K4Oi-`_$-7bHV6d~+@MDqQGj)!T@?
z6ZHca)P(`cOJ4hX9QB7}bl@r$nz&TAkDfD}&rZf~eHhhUKx>`9?R&65|7hcD6st{H
zk5$+936on=39_yE#tpoTENV~9cx>noCZI)1T%|<eBTD*lX5mD=-57>Yte-NMU+$Ik
zw#~8(!8esc@GeZ>G7w=Tf8CI5)P$?B<MDQNG5d}Q{ru^nXik%pBK$0feD;}H{3aGH
z+x%%6<=P*_ZL_XLQ3LL(BW*jM44!$^I$ft@+IXdhD1(3cW}qry@LTY$k&!RoHyt-M
z3Rp+L*U_U<z*h8E0ap)K0Z0t+#Y(DU$-aoG9g(cbtvK<a=7#*1Q$F8^;Lehg9VhpS
zkSs@oL-$`{AFM5gTfFHMTWq}}-GGvPktVDk=hoc&kioSMjoMhM)Y>bVR=cg)n3};d
zZclYBoZBSqi{2-LsG0V%124iVO017drPT&bP&XD(96kS|qS;5a?#r<!Lvez<7L1ps
zea5*7_yeiiZ|h9B<X$d*ab>?V^UU{7`S+8Qw|OK7)*1FX$}bw2-u^mloR&EKCNA2E
zIpzhKy}=0gBWe2{Ez~>;?$qPFXY5}R&6O}sh|1=At1a!BAd*zk-}BH8I}%$W?0WuA
zX%*r|WNn~zfbTuq_&XGdk`(eio!MIQESw!(8QS{_`Zr5oE+cMxVR73hu9Rye5N^JY
zRY>+xjw0hu$9Dd7T+Oi{wwG~}FIu!SnqqhPP*LcKvG<JjOvOkY;p-$Zqr38O4|Z*3
zROFWY5}Nsx@tq;|F`A*PX-B@@rm6!!kQy5{FrIv{(emoy=;EOUr^1<Hs-s^QD6SsT
ziT#s5z|#3^Y5f8T;H%&pIdv^91-(`94aB^k$PKu}u0n2>V1iW;1F+kX&0D`CHvs=2
zBR79Ey%uHwb~<4!9s@^-0^JOS2FD8kWd07?fbX7uf;M0i2Ne9bKpSANM0i@c0L;aS
z2=ExvY4Aik;bH}0GvLE3@UN{Kcy<M-5dZ)XOTY#&_9A3~O&x$5aB-lbnZd-tE!AQf
z)B$E#IKlxkKd*<~0E7JR20t>}R(z{|P0K3AuEK8q;38nre}HY)<>&to*aidafxqmR
z<nckn`5yq=01yugU_LDPK8|cv#^7P%5C{Tn=xAWR#sL2+D6E*|F`zu~p##=j5QA|*
zCi(v&z*h30vQStut1lrpIA9Vc+x+ojkUbtk34rY!pf7k}F~dPX3KW1T8!e85fdIe=
z0IWcO3s?f8Kq^Hcvp59IKrxgLY{q!-(V!@(549;S2JG-+OKDI{9t;McKwv-+AuDJE
z9(Yedb#Y+eOKpI%pkMMcut=kUsU8YAQE_rC4h{-DgdqU?|56zU3PF*vFN`=;4+~8E
z5H0|j`oUPB00ArzezD9nGP41WL1qw$0YhO>pEywGHy{Ke5(UW*)Dr}Jk+EAcOa~1O
zYGSE6c}}3}Ai!|TbM~{Tr3WDJOHBalJ{hxtr~;Hj(+P>e%5<&7fLhoJm&h$(0R{$!
zO=ce?2S3|i5fHL4ftH~6pRGVdA(1e0-f{!tcyahjfCOPl1|T{3J&<_~<N}h3Re>xI
zkW8!uvaBr21SAiu1H^0cJ7@!vjg<h69FN3-IRvyqGO$h%2dEZVF2E^?UlmqlVkL%{
z0_Gh?j-lsOfjrks`B0gk0g?-7R-yPOTtnWVSN1wtAXiH#h6<bvDFC>}3}9bNX%=wo
zGF)N_fXJmZfUJ|@64>%S46euI6NkJ<1O6<%K-)I38j@cez}zf7LEAg|t_yl0@8-)u
z4GE50?r~|`$E|=g;3Oa`qoDjLXzwStM?NaRDurHF?RnsRWmHN4HC!f33CP|O9lwgM
zzI(``R(lov{O%YbFALg3SeU_4P#*|c^nWY{aG_ttF#oX_QUshxpdb`WORpk)m9qVd
z#8ukkuS94_D??O=7A=_t2z9VZmKgS{C^>JX?9zd@@{Kj(r*5F+<N~}MemNPyf!hXH
zaL`|H>@S<S@K~TE{N)oTjAqLlwC2>CH;-|TF}fW%-4;rhdlT#;*~;{B0!0B7`Wx<T
z>7>%E7EzPwVKQjgz`YT-*Ei@vAkBvTJDJD-)RM@<`Cm)@YKzGd>z6);N2uCbLOv>B
zQ-DMla&>V!1za}B61M!28E}XEHOKPci*LvsXCH8xadvPbIoO)R-7q5JB4{D_2@=WK
zRRV!<{Fx=<<YFTLLjczq%TpHM1M=0ioGq>4=4KZ5pqJ$qKuL1*woXI^=tKK`3KD1}
z8XPhxfVm^Z(Bk`%XnrJ;UtpD_y1Q6e!@yB34g;<b%Rg{t;=m4L4gZ-2R4#HJoVc8Z
z!lUqiq~UOXmc!w2V5vgCUv+V!kdM&XGzdjrn+9!Kzomf}pd_wM!$64f@8!^t8~vIz
zBw)hYG&CA0F~65X;Y5L6u{I5IdHOvKDGE7ctWCp0+aC1$IbPs4_2)Q%hsz)Bp)r5(
z1xz2b<^J9t(6RpL1A_u*<C=27sRC>-zt_cxf^%_A8jy~)yayCR*!%Bwu~^YR+QW+d
zSq=wzcdV(4gM2X7rs3BqCl2`w{N67({{CnW4=ELE%i(}>u{I4%(wa0B77d+gzx9E_
zBB4X`_cY)dy0%{wZk;(26NkJFeyfWVM}n)%+BEc^d=|$8cc-=G!~ydDdm0LfhPD{!
z_fxKbha2Pv@p~H3t^O#7!a#l*Ysz7Pp1U>;|Ht|OPJTddUQ-TS{@0Ed+$(_x)!K59
zezhhI^Cw?`#S>CLpx>{t;laZEl}2(gvvsg?A^)c&+P0oR;{f+Uf|C=p(USEC6{0mz
eE1->*tUtJt%v?z1tqq08h+$y7yoyJZVE+e>dyoDA
literal 0
HcmV?d00001
diff --git a/2_psrs_cdist/solutions/A1/strong_scaling_symmetric/nodes_1.out b/2_psrs_cdist/solutions/A1/strong_scaling_symmetric/nodes_1.out
new file mode 100644
index 0000000..f4de03f
--- /dev/null
+++ b/2_psrs_cdist/solutions/A1/strong_scaling_symmetric/nodes_1.out
@@ -0,0 +1,136 @@
+Run 1
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/1]: Local data chunk has shape [50000, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [50000, 50000].
+Process-averaged run time: 5.940975139848888 s
+Run 2
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/1]: Local data chunk has shape [50000, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [50000, 50000].
+Process-averaged run time: 5.898665787652135 s
+Run 3
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/1]: Local data chunk has shape [50000, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [50000, 50000].
+Process-averaged run time: 5.895133529789746 s
+Run 4
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/1]: Local data chunk has shape [50000, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [50000, 50000].
+Process-averaged run time: 5.907652365043759 s
+Run 5
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/1]: Local data chunk has shape [50000, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [50000, 50000].
+Process-averaged run time: 5.929541239514947 s
+Run 6
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/1]: Local data chunk has shape [50000, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [50000, 50000].
+Process-averaged run time: 5.893667569383979 s
+Run 7
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/1]: Local data chunk has shape [50000, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [50000, 50000].
+Process-averaged run time: 5.90837056748569 s
+Run 8
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/1]: Local data chunk has shape [50000, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [50000, 50000].
+Process-averaged run time: 5.9261263478547335 s
+Run 9
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/1]: Local data chunk has shape [50000, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [50000, 50000].
+Process-averaged run time: 5.914638973772526 s
+Run 10
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/1]: Local data chunk has shape [50000, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [50000, 50000].
+Process-averaged run time: 5.895798914134502 s
+Run 11
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/1]: Local data chunk has shape [50000, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [50000, 50000].
+Process-averaged run time: 5.89725339692086 s
+
+============================= JOB FEEDBACK =============================
+
+NodeName=uc2n341
+Job ID: 24894108
+Cluster: uc2
+User/Group: ku4408/scc
+State: COMPLETED (exit code 0)
+Nodes: 1
+Cores per node: 40
+CPU Utilized: 00:01:32
+CPU Efficiency: 1.67% of 01:32:00 core-walltime
+Job Wall-clock time: 00:02:18
+Memory Utilized: 411.15 MB
+Memory Efficiency: 0.91% of 43.95 GB
diff --git a/2_psrs_cdist/solutions/A1/strong_scaling_symmetric/nodes_16.out b/2_psrs_cdist/solutions/A1/strong_scaling_symmetric/nodes_16.out
new file mode 100644
index 0000000..d00f7d3
--- /dev/null
+++ b/2_psrs_cdist/solutions/A1/strong_scaling_symmetric/nodes_16.out
@@ -0,0 +1,2743 @@
+Run 1
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [14/16]: Calculate diagonal blocks...
+Rank [8/16]: Calculate diagonal blocks...
+Rank [7/16]: Calculate diagonal blocks...
+Rank [10/16]: Calculate diagonal blocks...
+Rank [3/16]: Calculate diagonal blocks...
+Rank [5/16]: Calculate diagonal blocks...
+Rank [0/16]: Calculate diagonal blocks...
+Rank [12/16]: Calculate diagonal blocks...
+Rank [4/16]: Calculate diagonal blocks...
+Rank [11/16]: Calculate diagonal blocks...
+Rank [2/16]: Calculate diagonal blocks...
+Rank [15/16]: Calculate diagonal blocks...
+Rank [6/16]: Calculate diagonal blocks...
+Rank [13/16]: Calculate diagonal blocks...
+Rank [9/16]: Calculate diagonal blocks...
+Rank [1/16]: Calculate diagonal blocks...
+Rank [14/16]: Start tile-wise ring communication...
+Rank [14/16]: Starting iteration 1
+Rank [7/16]: Start tile-wise ring communication...
+Rank [7/16]: Starting iteration 1
+Rank [10/16]: Start tile-wise ring communication...
+Rank [10/16]: Starting iteration 1
+Rank [8/16]: Start tile-wise ring communication...
+Rank [8/16]: Starting iteration 1
+Rank [3/16]: Start tile-wise ring communication...
+Rank [3/16]: Starting iteration 1
+Rank [5/16]: Start tile-wise ring communication...
+Rank [5/16]: Starting iteration 1
+Rank [0/16]: Start tile-wise ring communication...
+Rank [0/16]: Starting iteration 1
+Rank [4/16]: Start tile-wise ring communication...
+Rank [4/16]: Starting iteration 1
+Rank [12/16]: Start tile-wise ring communication...
+Rank [12/16]: Starting iteration 1
+Rank [11/16]: Start tile-wise ring communication...
+Rank [11/16]: Starting iteration 1
+Rank [15/16]: Start tile-wise ring communication...
+Rank [15/16]: Starting iteration 1
+Rank [9/16]: Start tile-wise ring communication...
+Rank [9/16]: Starting iteration 1
+Rank [2/16]: Start tile-wise ring communication...
+Rank [2/16]: Starting iteration 1
+Rank [6/16]: Start tile-wise ring communication...
+Rank [6/16]: Starting iteration 1
+Rank [13/16]: Start tile-wise ring communication...
+Rank [13/16]: Starting iteration 1
+Rank [1/16]: Start tile-wise ring communication...
+Rank [1/16]: Starting iteration 1
+Rank [15/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 2
+Rank [13/16]: Starting iteration 2
+Rank [12/16]: Starting iteration 2
+Rank [11/16]: Starting iteration 2
+Rank [10/16]: Starting iteration 2
+Rank [9/16]: Starting iteration 2
+Rank [8/16]: Starting iteration 2
+Rank [7/16]: Starting iteration 2
+Rank [6/16]: Starting iteration 2
+Rank [5/16]: Starting iteration 2
+Rank [4/16]: Starting iteration 2
+Rank [3/16]: Starting iteration 2
+Rank [2/16]: Starting iteration 2
+Rank [0/16]: Starting iteration 2
+Rank [1/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 3
+Rank [15/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 3
+Rank [12/16]: Starting iteration 3
+Rank [11/16]: Starting iteration 3
+Rank [10/16]: Starting iteration 3
+Rank [9/16]: Starting iteration 3
+Rank [8/16]: Starting iteration 3
+Rank [7/16]: Starting iteration 3
+Rank [6/16]: Starting iteration 3
+Rank [5/16]: Starting iteration 3
+Rank [4/16]: Starting iteration 3
+Rank [0/16]: Starting iteration 3
+Rank [1/16]: Starting iteration 3
+Rank [2/16]: Starting iteration 3
+Rank [3/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 4
+Rank [14/16]: Starting iteration 4
+Rank [15/16]: Starting iteration 4
+Rank [10/16]: Starting iteration 4
+Rank [11/16]: Starting iteration 4
+Rank [12/16]: Starting iteration 4
+Rank [7/16]: Starting iteration 4
+Rank [8/16]: Starting iteration 4
+Rank [9/16]: Starting iteration 4
+Rank [4/16]: Starting iteration 4
+Rank [1/16]: Starting iteration 4
+Rank [2/16]: Starting iteration 4
+Rank [5/16]: Starting iteration 4
+Rank [6/16]: Starting iteration 4
+Rank [0/16]: Starting iteration 4
+Rank [3/16]: Starting iteration 4
+Rank [13/16]: Starting iteration 5
+Rank [14/16]: Starting iteration 5
+Rank [9/16]: Starting iteration 5
+Rank [12/16]: Starting iteration 5
+Rank [10/16]: Starting iteration 5
+Rank [15/16]: Starting iteration 5
+Rank [1/16]: Starting iteration 5
+Rank [8/16]: Starting iteration 5
+Rank [5/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 5
+Rank [2/16]: Starting iteration 5
+Rank [6/16]: Starting iteration 5
+Rank [0/16]: Starting iteration 5
+Rank [4/16]: Starting iteration 5
+Rank [7/16]: Starting iteration 5
+Rank [3/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 6
+Rank [15/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 6
+Rank [13/16]: Starting iteration 6
+Rank [10/16]: Starting iteration 6
+Rank [1/16]: Starting iteration 6
+Rank [2/16]: Starting iteration 6
+Rank [6/16]: Starting iteration 6
+Rank [7/16]: Starting iteration 6
+Rank [0/16]: Starting iteration 6
+Rank [5/16]: Starting iteration 6
+Rank [8/16]: Starting iteration 6
+Rank [14/16]: Starting iteration 6
+Rank [3/16]: Starting iteration 6
+Rank [4/16]: Starting iteration 6
+Rank [9/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 7
+Rank [13/16]: Starting iteration 7
+Rank [0/16]: Starting iteration 7
+Rank [5/16]: Starting iteration 7
+Rank [11/16]: Starting iteration 7
+Rank [6/16]: Starting iteration 7
+Rank [4/16]: Starting iteration 7
+Rank [10/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 7
+Rank [15/16]: Starting iteration 7
+Rank [1/16]: Starting iteration 7
+Rank [7/16]: Starting iteration 7
+Rank [2/16]: Starting iteration 7
+Rank [3/16]: Starting iteration 7
+Rank [8/16]: Starting iteration 7
+Rank [9/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 8
+Rank [15/16]: Starting iteration 8
+Rank [0/16]: Starting iteration 8
+Rank [7/16]: Starting iteration 8
+Rank [1/16]: Starting iteration 8
+Rank [8/16]: Starting iteration 8
+Rank [13/16]: Starting iteration 8
+Rank [6/16]: Starting iteration 8
+Rank [2/16]: Starting iteration 8
+Rank [4/16]: Starting iteration 8
+Rank [11/16]: Starting iteration 8
+Rank [10/16]: Starting iteration 8
+Rank [9/16]: Starting iteration 8
+Rank [5/16]: Starting iteration 8
+Rank [12/16]: Starting iteration 8
+Rank [3/16]: Starting iteration 8
+Rank [8/16]: [DONE]
+Rank [15/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 1.2271180103998631 s
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Run 2
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks...
+Rank [13/16]: Calculate diagonal blocks...
+Rank [2/16]: Calculate diagonal blocks...
+Rank [1/16]: Calculate diagonal blocks...
+Rank [15/16]: Calculate diagonal blocks...
+Rank [11/16]: Calculate diagonal blocks...
+Rank [8/16]: Calculate diagonal blocks...
+Rank [3/16]: Calculate diagonal blocks...
+Rank [6/16]: Calculate diagonal blocks...
+Rank [12/16]: Calculate diagonal blocks...
+Rank [9/16]: Calculate diagonal blocks...
+Rank [7/16]: Calculate diagonal blocks...
+Rank [5/16]: Calculate diagonal blocks...
+Rank [14/16]: Calculate diagonal blocks...
+Rank [4/16]: Calculate diagonal blocks...
+Rank [10/16]: Calculate diagonal blocks...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [0/16]: Starting iteration 1
+Rank [3/16]: Start tile-wise ring communication...
+Rank [3/16]: Starting iteration 1
+Rank [7/16]: Start tile-wise ring communication...
+Rank [7/16]: Starting iteration 1
+Rank [9/16]: Start tile-wise ring communication...
+Rank [9/16]: Starting iteration 1
+Rank [5/16]: Start tile-wise ring communication...
+Rank [5/16]: Starting iteration 1
+Rank [13/16]: Start tile-wise ring communication...
+Rank [13/16]: Starting iteration 1
+Rank [2/16]: Start tile-wise ring communication...
+Rank [2/16]: Starting iteration 1
+Rank [10/16]: Start tile-wise ring communication...
+Rank [10/16]: Starting iteration 1
+Rank [11/16]: Start tile-wise ring communication...
+Rank [11/16]: Starting iteration 1
+Rank [8/16]: Start tile-wise ring communication...
+Rank [8/16]: Starting iteration 1
+Rank [12/16]: Start tile-wise ring communication...
+Rank [12/16]: Starting iteration 1
+Rank [14/16]: Start tile-wise ring communication...
+Rank [14/16]: Starting iteration 1
+Rank [1/16]: Start tile-wise ring communication...
+Rank [1/16]: Starting iteration 1
+Rank [15/16]: Start tile-wise ring communication...
+Rank [15/16]: Starting iteration 1
+Rank [6/16]: Start tile-wise ring communication...
+Rank [6/16]: Starting iteration 1
+Rank [4/16]: Start tile-wise ring communication...
+Rank [4/16]: Starting iteration 1
+Rank [15/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 2
+Rank [13/16]: Starting iteration 2
+Rank [12/16]: Starting iteration 2
+Rank [11/16]: Starting iteration 2
+Rank [10/16]: Starting iteration 2
+Rank [9/16]: Starting iteration 2
+Rank [8/16]: Starting iteration 2
+Rank [7/16]: Starting iteration 2
+Rank [6/16]: Starting iteration 2
+Rank [5/16]: Starting iteration 2
+Rank [4/16]: Starting iteration 2
+Rank [3/16]: Starting iteration 2
+Rank [2/16]: Starting iteration 2
+Rank [0/16]: Starting iteration 2
+Rank [1/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 3
+Rank [15/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 3
+Rank [12/16]: Starting iteration 3
+Rank [10/16]: Starting iteration 3
+Rank [11/16]: Starting iteration 3
+Rank [8/16]: Starting iteration 3
+Rank [9/16]: Starting iteration 3
+Rank [7/16]: Starting iteration 3
+Rank [6/16]: Starting iteration 3
+Rank [5/16]: Starting iteration 3
+Rank [4/16]: Starting iteration 3
+Rank [1/16]: Starting iteration 3
+Rank [0/16]: Starting iteration 3
+Rank [2/16]: Starting iteration 3
+Rank [3/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 4
+Rank [14/16]: Starting iteration 4
+Rank [15/16]: Starting iteration 4
+Rank [10/16]: Starting iteration 4
+Rank [11/16]: Starting iteration 4
+Rank [12/16]: Starting iteration 4
+Rank [7/16]: Starting iteration 4
+Rank [8/16]: Starting iteration 4
+Rank [9/16]: Starting iteration 4
+Rank [4/16]: Starting iteration 4
+Rank [1/16]: Starting iteration 4
+Rank [2/16]: Starting iteration 4
+Rank [5/16]: Starting iteration 4
+Rank [6/16]: Starting iteration 4
+Rank [0/16]: Starting iteration 4
+Rank [3/16]: Starting iteration 4
+Rank [13/16]: Starting iteration 5
+Rank [14/16]: Starting iteration 5
+Rank [10/16]: Starting iteration 5
+Rank [12/16]: Starting iteration 5
+Rank [9/16]: Starting iteration 5
+Rank [15/16]: Starting iteration 5
+Rank [1/16]: Starting iteration 5
+Rank [5/16]: Starting iteration 5
+Rank [8/16]: Starting iteration 5
+Rank [2/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 5
+Rank [6/16]: Starting iteration 5
+Rank [0/16]: Starting iteration 5
+Rank [4/16]: Starting iteration 5
+Rank [7/16]: Starting iteration 5
+Rank [3/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 6
+Rank [15/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 6
+Rank [13/16]: Starting iteration 6
+Rank [1/16]: Starting iteration 6
+Rank [10/16]: Starting iteration 6
+Rank [2/16]: Starting iteration 6
+Rank [7/16]: Starting iteration 6
+Rank [6/16]: Starting iteration 6
+Rank [0/16]: Starting iteration 6
+Rank [5/16]: Starting iteration 6
+Rank [14/16]: Starting iteration 6
+Rank [8/16]: Starting iteration 6
+Rank [3/16]: Starting iteration 6
+Rank [4/16]: Starting iteration 6
+Rank [9/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 7
+Rank [0/16]: Starting iteration 7
+Rank [6/16]: Starting iteration 7
+Rank [5/16]: Starting iteration 7
+Rank [11/16]: Starting iteration 7
+Rank [13/16]: Starting iteration 7
+Rank [15/16]: Starting iteration 7
+Rank [1/16]: Starting iteration 7
+Rank [7/16]: Starting iteration 7
+Rank [3/16]: Starting iteration 7
+Rank [9/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 7
+Rank [4/16]: Starting iteration 7
+Rank [10/16]: Starting iteration 7
+Rank [2/16]: Starting iteration 7
+Rank [8/16]: Starting iteration 7
+Rank [5/16]: Starting iteration 8
+Rank [14/16]: Starting iteration 8
+Rank [12/16]: Starting iteration 8
+Rank [0/16]: Starting iteration 8
+Rank [7/16]: Starting iteration 8
+Rank [15/16]: Starting iteration 8
+Rank [1/16]: Starting iteration 8
+Rank [10/16]: Starting iteration 8
+Rank [8/16]: Starting iteration 8
+Rank [3/16]: Starting iteration 8
+Rank [13/16]: Starting iteration 8
+Rank [6/16]: Starting iteration 8
+Rank [2/16]: Starting iteration 8
+Rank [11/16]: Starting iteration 8
+Rank [4/16]: Starting iteration 8
+Rank [9/16]: Starting iteration 8
+Rank [15/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.985500194132328 s
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Run 3
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks...
+Rank [15/16]: Calculate diagonal blocks...
+Rank [3/16]: Calculate diagonal blocks...
+Rank [5/16]: Calculate diagonal blocks...
+Rank [11/16]: Calculate diagonal blocks...
+Rank [8/16]: Calculate diagonal blocks...
+Rank [10/16]: Calculate diagonal blocks...
+Rank [2/16]: Calculate diagonal blocks...
+Rank [12/16]: Calculate diagonal blocks...
+Rank [9/16]: Calculate diagonal blocks...
+Rank [13/16]: Calculate diagonal blocks...
+Rank [4/16]: Calculate diagonal blocks...
+Rank [1/16]: Calculate diagonal blocks...
+Rank [14/16]: Calculate diagonal blocks...
+Rank [7/16]: Calculate diagonal blocks...
+Rank [6/16]: Calculate diagonal blocks...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [0/16]: Starting iteration 1
+Rank [15/16]: Start tile-wise ring communication...
+Rank [15/16]: Starting iteration 1
+Rank [3/16]: Start tile-wise ring communication...
+Rank [3/16]: Starting iteration 1
+Rank [5/16]: Start tile-wise ring communication...
+Rank [5/16]: Starting iteration 1
+Rank [11/16]: Start tile-wise ring communication...
+Rank [11/16]: Starting iteration 1
+Rank [8/16]: Start tile-wise ring communication...
+Rank [8/16]: Starting iteration 1
+Rank [10/16]: Start tile-wise ring communication...
+Rank [10/16]: Starting iteration 1
+Rank [2/16]: Start tile-wise ring communication...
+Rank [2/16]: Starting iteration 1
+Rank [9/16]: Start tile-wise ring communication...
+Rank [9/16]: Starting iteration 1
+Rank [1/16]: Start tile-wise ring communication...
+Rank [1/16]: Starting iteration 1
+Rank [7/16]: Start tile-wise ring communication...
+Rank [7/16]: Starting iteration 1
+Rank [13/16]: Start tile-wise ring communication...
+Rank [13/16]: Starting iteration 1
+Rank [4/16]: Start tile-wise ring communication...
+Rank [4/16]: Starting iteration 1
+Rank [14/16]: Start tile-wise ring communication...
+Rank [14/16]: Starting iteration 1
+Rank [6/16]: Start tile-wise ring communication...
+Rank [6/16]: Starting iteration 1
+Rank [12/16]: Start tile-wise ring communication...
+Rank [12/16]: Starting iteration 1
+Rank [15/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 2
+Rank [13/16]: Starting iteration 2
+Rank [12/16]: Starting iteration 2
+Rank [11/16]: Starting iteration 2
+Rank [10/16]: Starting iteration 2
+Rank [9/16]: Starting iteration 2
+Rank [8/16]: Starting iteration 2
+Rank [7/16]: Starting iteration 2
+Rank [6/16]: Starting iteration 2
+Rank [5/16]: Starting iteration 2
+Rank [4/16]: Starting iteration 2
+Rank [3/16]: Starting iteration 2
+Rank [2/16]: Starting iteration 2
+Rank [0/16]: Starting iteration 2
+Rank [1/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 3
+Rank [15/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 3
+Rank [12/16]: Starting iteration 3
+Rank [11/16]: Starting iteration 3
+Rank [10/16]: Starting iteration 3
+Rank [8/16]: Starting iteration 3
+Rank [9/16]: Starting iteration 3
+Rank [7/16]: Starting iteration 3
+Rank [6/16]: Starting iteration 3
+Rank [5/16]: Starting iteration 3
+Rank [4/16]: Starting iteration 3
+Rank [1/16]: Starting iteration 3
+Rank [0/16]: Starting iteration 3
+Rank [2/16]: Starting iteration 3
+Rank [3/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 4
+Rank [14/16]: Starting iteration 4
+Rank [15/16]: Starting iteration 4
+Rank [10/16]: Starting iteration 4
+Rank [11/16]: Starting iteration 4
+Rank [12/16]: Starting iteration 4
+Rank [7/16]: Starting iteration 4
+Rank [8/16]: Starting iteration 4
+Rank [9/16]: Starting iteration 4
+Rank [1/16]: Starting iteration 4
+Rank [4/16]: Starting iteration 4
+Rank [2/16]: Starting iteration 4
+Rank [5/16]: Starting iteration 4
+Rank [6/16]: Starting iteration 4
+Rank [0/16]: Starting iteration 4
+Rank [3/16]: Starting iteration 4
+Rank [13/16]: Starting iteration 5
+Rank [14/16]: Starting iteration 5
+Rank [9/16]: Starting iteration 5
+Rank [12/16]: Starting iteration 5
+Rank [10/16]: Starting iteration 5
+Rank [15/16]: Starting iteration 5
+Rank [1/16]: Starting iteration 5
+Rank [5/16]: Starting iteration 5
+Rank [8/16]: Starting iteration 5
+Rank [2/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 5
+Rank [6/16]: Starting iteration 5
+Rank [0/16]: Starting iteration 5
+Rank [4/16]: Starting iteration 5
+Rank [7/16]: Starting iteration 5
+Rank [3/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 6
+Rank [15/16]: Starting iteration 6
+Rank [13/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 6
+Rank [10/16]: Starting iteration 6
+Rank [1/16]: Starting iteration 6
+Rank [6/16]: Starting iteration 6
+Rank [2/16]: Starting iteration 6
+Rank [7/16]: Starting iteration 6
+Rank [0/16]: Starting iteration 6
+Rank [5/16]: Starting iteration 6
+Rank [14/16]: Starting iteration 6
+Rank [8/16]: Starting iteration 6
+Rank [3/16]: Starting iteration 6
+Rank [4/16]: Starting iteration 6
+Rank [9/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 7
+Rank [13/16]: Starting iteration 7
+Rank [0/16]: Starting iteration 7
+Rank [5/16]: Starting iteration 7
+Rank [6/16]: Starting iteration 7
+Rank [11/16]: Starting iteration 7
+Rank [4/16]: Starting iteration 7
+Rank [15/16]: Starting iteration 7
+Rank [10/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 7
+Rank [1/16]: Starting iteration 7
+Rank [7/16]: Starting iteration 7
+Rank [3/16]: Starting iteration 7
+Rank [9/16]: Starting iteration 7
+Rank [2/16]: Starting iteration 7
+Rank [8/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 8
+Rank [15/16]: Starting iteration 8
+Rank [0/16]: Starting iteration 8
+Rank [7/16]: Starting iteration 8
+Rank [2/16]: Starting iteration 8
+Rank [9/16]: Starting iteration 8
+Rank [5/16]: Starting iteration 8
+Rank [12/16]: Starting iteration 8
+Rank [4/16]: Starting iteration 8
+Rank [1/16]: Starting iteration 8
+Rank [8/16]: Starting iteration 8
+Rank [10/16]: Starting iteration 8
+Rank [13/16]: Starting iteration 8
+Rank [11/16]: Starting iteration 8
+Rank [3/16]: Starting iteration 8
+Rank [6/16]: Starting iteration 8
+Rank [15/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.9802435667952523 s
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Run 4
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks...
+Rank [11/16]: Calculate diagonal blocks...
+Rank [9/16]: Calculate diagonal blocks...
+Rank [2/16]: Calculate diagonal blocks...
+Rank [14/16]: Calculate diagonal blocks...
+Rank [5/16]: Calculate diagonal blocks...
+Rank [1/16]: Calculate diagonal blocks...
+Rank [4/16]: Calculate diagonal blocks...
+Rank [3/16]: Calculate diagonal blocks...
+Rank [8/16]: Calculate diagonal blocks...
+Rank [15/16]: Calculate diagonal blocks...
+Rank [12/16]: Calculate diagonal blocks...
+Rank [6/16]: Calculate diagonal blocks...
+Rank [13/16]: Calculate diagonal blocks...
+Rank [7/16]: Calculate diagonal blocks...
+Rank [10/16]: Calculate diagonal blocks...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [0/16]: Starting iteration 1
+Rank [7/16]: Start tile-wise ring communication...
+Rank [7/16]: Starting iteration 1
+Rank [4/16]: Start tile-wise ring communication...
+Rank [4/16]: Starting iteration 1
+Rank [3/16]: Start tile-wise ring communication...
+Rank [3/16]: Starting iteration 1
+Rank [5/16]: Start tile-wise ring communication...
+Rank [5/16]: Starting iteration 1
+Rank [15/16]: Start tile-wise ring communication...
+Rank [15/16]: Starting iteration 1
+Rank [6/16]: Start tile-wise ring communication...
+Rank [6/16]: Starting iteration 1
+Rank [11/16]: Start tile-wise ring communication...
+Rank [11/16]: Starting iteration 1
+Rank [13/16]: Start tile-wise ring communication...
+Rank [13/16]: Starting iteration 1
+Rank [9/16]: Start tile-wise ring communication...
+Rank [9/16]: Starting iteration 1
+Rank [10/16]: Start tile-wise ring communication...
+Rank [10/16]: Starting iteration 1
+Rank [1/16]: Start tile-wise ring communication...
+Rank [1/16]: Starting iteration 1
+Rank [2/16]: Start tile-wise ring communication...
+Rank [2/16]: Starting iteration 1
+Rank [14/16]: Start tile-wise ring communication...
+Rank [14/16]: Starting iteration 1
+Rank [8/16]: Start tile-wise ring communication...
+Rank [8/16]: Starting iteration 1
+Rank [12/16]: Start tile-wise ring communication...
+Rank [12/16]: Starting iteration 1
+Rank [15/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 2
+Rank [13/16]: Starting iteration 2
+Rank [12/16]: Starting iteration 2
+Rank [11/16]: Starting iteration 2
+Rank [10/16]: Starting iteration 2
+Rank [9/16]: Starting iteration 2
+Rank [8/16]: Starting iteration 2
+Rank [7/16]: Starting iteration 2
+Rank [6/16]: Starting iteration 2
+Rank [5/16]: Starting iteration 2
+Rank [4/16]: Starting iteration 2
+Rank [3/16]: Starting iteration 2
+Rank [2/16]: Starting iteration 2
+Rank [0/16]: Starting iteration 2
+Rank [1/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 3
+Rank [15/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 3
+Rank [12/16]: Starting iteration 3
+Rank [11/16]: Starting iteration 3
+Rank [10/16]: Starting iteration 3
+Rank [8/16]: Starting iteration 3
+Rank [9/16]: Starting iteration 3
+Rank [7/16]: Starting iteration 3
+Rank [6/16]: Starting iteration 3
+Rank [5/16]: Starting iteration 3
+Rank [4/16]: Starting iteration 3
+Rank [0/16]: Starting iteration 3
+Rank [1/16]: Starting iteration 3
+Rank [2/16]: Starting iteration 3
+Rank [3/16]: Starting iteration 3
+Rank [14/16]: Starting iteration 4
+Rank [13/16]: Starting iteration 4
+Rank [15/16]: Starting iteration 4
+Rank [10/16]: Starting iteration 4
+Rank [11/16]: Starting iteration 4
+Rank [12/16]: Starting iteration 4
+Rank [7/16]: Starting iteration 4
+Rank [8/16]: Starting iteration 4
+Rank [9/16]: Starting iteration 4
+Rank [1/16]: Starting iteration 4
+Rank [2/16]: Starting iteration 4
+Rank [5/16]: Starting iteration 4
+Rank [4/16]: Starting iteration 4
+Rank [6/16]: Starting iteration 4
+Rank [0/16]: Starting iteration 4
+Rank [3/16]: Starting iteration 4
+Rank [13/16]: Starting iteration 5
+Rank [14/16]: Starting iteration 5
+Rank [9/16]: Starting iteration 5
+Rank [10/16]: Starting iteration 5
+Rank [12/16]: Starting iteration 5
+Rank [15/16]: Starting iteration 5
+Rank [1/16]: Starting iteration 5
+Rank [5/16]: Starting iteration 5
+Rank [8/16]: Starting iteration 5
+Rank [2/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 5
+Rank [6/16]: Starting iteration 5
+Rank [0/16]: Starting iteration 5
+Rank [4/16]: Starting iteration 5
+Rank [7/16]: Starting iteration 5
+Rank [3/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 6
+Rank [15/16]: Starting iteration 6
+Rank [13/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 6
+Rank [1/16]: Starting iteration 6
+Rank [10/16]: Starting iteration 6
+Rank [2/16]: Starting iteration 6
+Rank [7/16]: Starting iteration 6
+Rank [6/16]: Starting iteration 6
+Rank [0/16]: Starting iteration 6
+Rank [5/16]: Starting iteration 6
+Rank [14/16]: Starting iteration 6
+Rank [8/16]: Starting iteration 6
+Rank [3/16]: Starting iteration 6
+Rank [4/16]: Starting iteration 6
+Rank [9/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 7
+Rank [13/16]: Starting iteration 7
+Rank [0/16]: Starting iteration 7
+Rank [5/16]: Starting iteration 7
+Rank [11/16]: Starting iteration 7
+Rank [6/16]: Starting iteration 7
+Rank [15/16]: Starting iteration 7
+Rank [10/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 7
+Rank [4/16]: Starting iteration 7
+Rank [1/16]: Starting iteration 7
+Rank [7/16]: Starting iteration 7
+Rank [3/16]: Starting iteration 7
+Rank [9/16]: Starting iteration 7
+Rank [2/16]: Starting iteration 7
+Rank [8/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 8
+Rank [15/16]: Starting iteration 8
+Rank [0/16]: Starting iteration 8
+Rank [7/16]: Starting iteration 8
+Rank [2/16]: Starting iteration 8
+Rank [9/16]: Starting iteration 8
+Rank [11/16]: Starting iteration 8
+Rank [13/16]: Starting iteration 8
+Rank [4/16]: Starting iteration 8
+Rank [6/16]: Starting iteration 8
+Rank [5/16]: Starting iteration 8
+Rank [12/16]: Starting iteration 8
+Rank [1/16]: Starting iteration 8
+Rank [8/16]: Starting iteration 8
+Rank [10/16]: Starting iteration 8
+Rank [3/16]: Starting iteration 8
+Rank [15/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.9844902949989773 s
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Run 5
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks...
+Rank [15/16]: Calculate diagonal blocks...
+Rank [13/16]: Calculate diagonal blocks...
+Rank [10/16]: Calculate diagonal blocks...
+Rank [2/16]: Calculate diagonal blocks...
+Rank [5/16]: Calculate diagonal blocks...
+Rank [7/16]: Calculate diagonal blocks...
+Rank [3/16]: Calculate diagonal blocks...
+Rank [6/16]: Calculate diagonal blocks...
+Rank [9/16]: Calculate diagonal blocks...
+Rank [12/16]: Calculate diagonal blocks...
+Rank [8/16]: Calculate diagonal blocks...
+Rank [14/16]: Calculate diagonal blocks...
+Rank [1/16]: Calculate diagonal blocks...
+Rank [11/16]: Calculate diagonal blocks...
+Rank [4/16]: Calculate diagonal blocks...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [0/16]: Starting iteration 1
+Rank [7/16]: Start tile-wise ring communication...
+Rank [7/16]: Starting iteration 1
+Rank [9/16]: Start tile-wise ring communication...
+Rank [9/16]: Starting iteration 1
+Rank [3/16]: Start tile-wise ring communication...
+Rank [3/16]: Starting iteration 1
+Rank [15/16]: Start tile-wise ring communication...
+Rank [15/16]: Starting iteration 1
+Rank [8/16]: Start tile-wise ring communication...
+Rank [8/16]: Starting iteration 1
+Rank [11/16]: Start tile-wise ring communication...
+Rank [11/16]: Starting iteration 1
+Rank [10/16]: Start tile-wise ring communication...
+Rank [10/16]: Starting iteration 1
+Rank [5/16]: Start tile-wise ring communication...
+Rank [5/16]: Starting iteration 1
+Rank [6/16]: Start tile-wise ring communication...
+Rank [6/16]: Starting iteration 1
+Rank [13/16]: Start tile-wise ring communication...
+Rank [13/16]: Starting iteration 1
+Rank [12/16]: Start tile-wise ring communication...
+Rank [12/16]: Starting iteration 1
+Rank [1/16]: Start tile-wise ring communication...
+Rank [1/16]: Starting iteration 1
+Rank [4/16]: Start tile-wise ring communication...
+Rank [4/16]: Starting iteration 1
+Rank [2/16]: Start tile-wise ring communication...
+Rank [2/16]: Starting iteration 1
+Rank [14/16]: Start tile-wise ring communication...
+Rank [14/16]: Starting iteration 1
+Rank [15/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 2
+Rank [13/16]: Starting iteration 2
+Rank [12/16]: Starting iteration 2
+Rank [11/16]: Starting iteration 2
+Rank [10/16]: Starting iteration 2
+Rank [9/16]: Starting iteration 2
+Rank [8/16]: Starting iteration 2
+Rank [7/16]: Starting iteration 2
+Rank [6/16]: Starting iteration 2
+Rank [5/16]: Starting iteration 2
+Rank [4/16]: Starting iteration 2
+Rank [3/16]: Starting iteration 2
+Rank [2/16]: Starting iteration 2
+Rank [1/16]: Starting iteration 2
+Rank [0/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 3
+Rank [15/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 3
+Rank [12/16]: Starting iteration 3
+Rank [10/16]: Starting iteration 3
+Rank [11/16]: Starting iteration 3
+Rank [8/16]: Starting iteration 3
+Rank [9/16]: Starting iteration 3
+Rank [7/16]: Starting iteration 3
+Rank [6/16]: Starting iteration 3
+Rank [4/16]: Starting iteration 3
+Rank [5/16]: Starting iteration 3
+Rank [1/16]: Starting iteration 3
+Rank [0/16]: Starting iteration 3
+Rank [2/16]: Starting iteration 3
+Rank [3/16]: Starting iteration 3
+Rank [14/16]: Starting iteration 4
+Rank [13/16]: Starting iteration 4
+Rank [15/16]: Starting iteration 4
+Rank [11/16]: Starting iteration 4
+Rank [10/16]: Starting iteration 4
+Rank [12/16]: Starting iteration 4
+Rank [8/16]: Starting iteration 4
+Rank [7/16]: Starting iteration 4
+Rank [9/16]: Starting iteration 4
+Rank [2/16]: Starting iteration 4
+Rank [5/16]: Starting iteration 4
+Rank [1/16]: Starting iteration 4
+Rank [4/16]: Starting iteration 4
+Rank [6/16]: Starting iteration 4
+Rank [0/16]: Starting iteration 4
+Rank [3/16]: Starting iteration 4
+Rank [13/16]: Starting iteration 5
+Rank [14/16]: Starting iteration 5
+Rank [9/16]: Starting iteration 5
+Rank [12/16]: Starting iteration 5
+Rank [10/16]: Starting iteration 5
+Rank [15/16]: Starting iteration 5
+Rank [1/16]: Starting iteration 5
+Rank [5/16]: Starting iteration 5
+Rank [8/16]: Starting iteration 5
+Rank [2/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 5
+Rank [6/16]: Starting iteration 5
+Rank [0/16]: Starting iteration 5
+Rank [4/16]: Starting iteration 5
+Rank [7/16]: Starting iteration 5
+Rank [3/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 6
+Rank [15/16]: Starting iteration 6
+Rank [13/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 6
+Rank [1/16]: Starting iteration 6
+Rank [10/16]: Starting iteration 6
+Rank [6/16]: Starting iteration 6
+Rank [2/16]: Starting iteration 6
+Rank [7/16]: Starting iteration 6
+Rank [0/16]: Starting iteration 6
+Rank [5/16]: Starting iteration 6
+Rank [8/16]: Starting iteration 6
+Rank [14/16]: Starting iteration 6
+Rank [3/16]: Starting iteration 6
+Rank [4/16]: Starting iteration 6
+Rank [9/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 7
+Rank [13/16]: Starting iteration 7
+Rank [0/16]: Starting iteration 7
+Rank [5/16]: Starting iteration 7
+Rank [11/16]: Starting iteration 7
+Rank [6/16]: Starting iteration 7
+Rank [15/16]: Starting iteration 7
+Rank [4/16]: Starting iteration 7
+Rank [10/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 7
+Rank [1/16]: Starting iteration 7
+Rank [7/16]: Starting iteration 7
+Rank [3/16]: Starting iteration 7
+Rank [9/16]: Starting iteration 7
+Rank [2/16]: Starting iteration 7
+Rank [8/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 8
+Rank [15/16]: Starting iteration 8
+Rank [0/16]: Starting iteration 8
+Rank [7/16]: Starting iteration 8
+Rank [5/16]: Starting iteration 8
+Rank [12/16]: Starting iteration 8
+Rank [13/16]: Starting iteration 8
+Rank [6/16]: Starting iteration 8
+Rank [1/16]: Starting iteration 8
+Rank [4/16]: Starting iteration 8
+Rank [2/16]: Starting iteration 8
+Rank [8/16]: Starting iteration 8
+Rank [11/16]: Starting iteration 8
+Rank [10/16]: Starting iteration 8
+Rank [3/16]: Starting iteration 8
+Rank [9/16]: Starting iteration 8
+Rank [15/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.984958013985306 s
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Run 6
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks...
+Rank [2/16]: Calculate diagonal blocks...
+Rank [6/16]: Calculate diagonal blocks...
+Rank [3/16]: Calculate diagonal blocks...
+Rank [12/16]: Calculate diagonal blocks...
+Rank [15/16]: Calculate diagonal blocks...
+Rank [1/16]: Calculate diagonal blocks...
+Rank [10/16]: Calculate diagonal blocks...
+Rank [14/16]: Calculate diagonal blocks...
+Rank [4/16]: Calculate diagonal blocks...
+Rank [7/16]: Calculate diagonal blocks...
+Rank [9/16]: Calculate diagonal blocks...
+Rank [8/16]: Calculate diagonal blocks...
+Rank [11/16]: Calculate diagonal blocks...
+Rank [13/16]: Calculate diagonal blocks...
+Rank [5/16]: Calculate diagonal blocks...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [0/16]: Starting iteration 1
+Rank [7/16]: Start tile-wise ring communication...
+Rank [7/16]: Starting iteration 1
+Rank [11/16]: Start tile-wise ring communication...
+Rank [11/16]: Starting iteration 1
+Rank [2/16]: Start tile-wise ring communication...
+Rank [2/16]: Starting iteration 1
+Rank [6/16]: Start tile-wise ring communication...
+Rank [6/16]: Starting iteration 1
+Rank [9/16]: Start tile-wise ring communication...
+Rank [9/16]: Starting iteration 1
+Rank [8/16]: Start tile-wise ring communication...
+Rank [8/16]: Starting iteration 1
+Rank [3/16]: Start tile-wise ring communication...
+Rank [3/16]: Starting iteration 1
+Rank [1/16]: Start tile-wise ring communication...
+Rank [1/16]: Starting iteration 1
+Rank [10/16]: Start tile-wise ring communication...
+Rank [10/16]: Starting iteration 1
+Rank [15/16]: Start tile-wise ring communication...
+Rank [15/16]: Starting iteration 1
+Rank [5/16]: Start tile-wise ring communication...
+Rank [5/16]: Starting iteration 1
+Rank [14/16]: Start tile-wise ring communication...
+Rank [14/16]: Starting iteration 1
+Rank [4/16]: Start tile-wise ring communication...
+Rank [4/16]: Starting iteration 1
+Rank [13/16]: Start tile-wise ring communication...
+Rank [13/16]: Starting iteration 1
+Rank [12/16]: Start tile-wise ring communication...
+Rank [12/16]: Starting iteration 1
+Rank [15/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 2
+Rank [13/16]: Starting iteration 2
+Rank [12/16]: Starting iteration 2
+Rank [11/16]: Starting iteration 2
+Rank [10/16]: Starting iteration 2
+Rank [9/16]: Starting iteration 2
+Rank [8/16]: Starting iteration 2
+Rank [7/16]: Starting iteration 2
+Rank [6/16]: Starting iteration 2
+Rank [5/16]: Starting iteration 2
+Rank [4/16]: Starting iteration 2
+Rank [3/16]: Starting iteration 2
+Rank [2/16]: Starting iteration 2
+Rank [0/16]: Starting iteration 2
+Rank [1/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 3
+Rank [15/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 3
+Rank [12/16]: Starting iteration 3
+Rank [10/16]: Starting iteration 3
+Rank [11/16]: Starting iteration 3
+Rank [8/16]: Starting iteration 3
+Rank [9/16]: Starting iteration 3
+Rank [7/16]: Starting iteration 3
+Rank [6/16]: Starting iteration 3
+Rank [4/16]: Starting iteration 3
+Rank [5/16]: Starting iteration 3
+Rank [0/16]: Starting iteration 3
+Rank [2/16]: Starting iteration 3
+Rank [1/16]: Starting iteration 3
+Rank [3/16]: Starting iteration 3
+Rank [14/16]: Starting iteration 4
+Rank [13/16]: Starting iteration 4
+Rank [15/16]: Starting iteration 4
+Rank [10/16]: Starting iteration 4
+Rank [11/16]: Starting iteration 4
+Rank [12/16]: Starting iteration 4
+Rank [8/16]: Starting iteration 4
+Rank [7/16]: Starting iteration 4
+Rank [9/16]: Starting iteration 4
+Rank [2/16]: Starting iteration 4
+Rank [1/16]: Starting iteration 4
+Rank [4/16]: Starting iteration 4
+Rank [5/16]: Starting iteration 4
+Rank [6/16]: Starting iteration 4
+Rank [0/16]: Starting iteration 4
+Rank [3/16]: Starting iteration 4
+Rank [13/16]: Starting iteration 5
+Rank [14/16]: Starting iteration 5
+Rank [10/16]: Starting iteration 5
+Rank [12/16]: Starting iteration 5
+Rank [9/16]: Starting iteration 5
+Rank [15/16]: Starting iteration 5
+Rank [1/16]: Starting iteration 5
+Rank [5/16]: Starting iteration 5
+Rank [8/16]: Starting iteration 5
+Rank [2/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 5
+Rank [6/16]: Starting iteration 5
+Rank [0/16]: Starting iteration 5
+Rank [4/16]: Starting iteration 5
+Rank [7/16]: Starting iteration 5
+Rank [3/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 6
+Rank [15/16]: Starting iteration 6
+Rank [13/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 6
+Rank [1/16]: Starting iteration 6
+Rank [10/16]: Starting iteration 6
+Rank [6/16]: Starting iteration 6
+Rank [2/16]: Starting iteration 6
+Rank [7/16]: Starting iteration 6
+Rank [0/16]: Starting iteration 6
+Rank [5/16]: Starting iteration 6
+Rank [8/16]: Starting iteration 6
+Rank [14/16]: Starting iteration 6
+Rank [3/16]: Starting iteration 6
+Rank [4/16]: Starting iteration 6
+Rank [9/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 7
+Rank [13/16]: Starting iteration 7
+Rank [0/16]: Starting iteration 7
+Rank [6/16]: Starting iteration 7
+Rank [11/16]: Starting iteration 7
+Rank [5/16]: Starting iteration 7
+Rank [4/16]: Starting iteration 7
+Rank [10/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 7
+Rank [15/16]: Starting iteration 7
+Rank [1/16]: Starting iteration 7
+Rank [7/16]: Starting iteration 7
+Rank [3/16]: Starting iteration 7
+Rank [9/16]: Starting iteration 7
+Rank [2/16]: Starting iteration 7
+Rank [8/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 8
+Rank [2/16]: Starting iteration 8
+Rank [15/16]: Starting iteration 8
+Rank [9/16]: Starting iteration 8
+Rank [0/16]: Starting iteration 8
+Rank [7/16]: Starting iteration 8
+Rank [1/16]: Starting iteration 8
+Rank [8/16]: Starting iteration 8
+Rank [10/16]: Starting iteration 8
+Rank [5/16]: Starting iteration 8
+Rank [3/16]: Starting iteration 8
+Rank [12/16]: Starting iteration 8
+Rank [4/16]: Starting iteration 8
+Rank [13/16]: Starting iteration 8
+Rank [11/16]: Starting iteration 8
+Rank [6/16]: Starting iteration 8
+Rank [15/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.9746054984279908 s
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Run 7
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks...
+Rank [1/16]: Calculate diagonal blocks...
+Rank [7/16]: Calculate diagonal blocks...
+Rank [10/16]: Calculate diagonal blocks...
+Rank [11/16]: Calculate diagonal blocks...
+Rank [5/16]: Calculate diagonal blocks...
+Rank [13/16]: Calculate diagonal blocks...
+Rank [2/16]: Calculate diagonal blocks...
+Rank [14/16]: Calculate diagonal blocks...
+Rank [9/16]: Calculate diagonal blocks...
+Rank [4/16]: Calculate diagonal blocks...
+Rank [6/16]: Calculate diagonal blocks...
+Rank [8/16]: Calculate diagonal blocks...
+Rank [15/16]: Calculate diagonal blocks...
+Rank [12/16]: Calculate diagonal blocks...
+Rank [3/16]: Calculate diagonal blocks...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [0/16]: Starting iteration 1
+Rank [7/16]: Start tile-wise ring communication...
+Rank [7/16]: Starting iteration 1
+Rank [13/16]: Start tile-wise ring communication...
+Rank [13/16]: Starting iteration 1
+Rank [15/16]: Start tile-wise ring communication...
+Rank [15/16]: Starting iteration 1
+Rank [9/16]: Start tile-wise ring communication...
+Rank [9/16]: Starting iteration 1
+Rank [10/16]: Start tile-wise ring communication...
+Rank [10/16]: Starting iteration 1
+Rank [11/16]: Start tile-wise ring communication...
+Rank [11/16]: Starting iteration 1
+Rank [5/16]: Start tile-wise ring communication...
+Rank [5/16]: Starting iteration 1
+Rank [12/16]: Start tile-wise ring communication...
+Rank [12/16]: Starting iteration 1
+Rank [2/16]: Start tile-wise ring communication...
+Rank [2/16]: Starting iteration 1
+Rank [4/16]: Start tile-wise ring communication...
+Rank [4/16]: Starting iteration 1
+Rank [6/16]: Start tile-wise ring communication...
+Rank [6/16]: Starting iteration 1
+Rank [3/16]: Start tile-wise ring communication...
+Rank [3/16]: Starting iteration 1
+Rank [1/16]: Start tile-wise ring communication...
+Rank [1/16]: Starting iteration 1
+Rank [14/16]: Start tile-wise ring communication...
+Rank [14/16]: Starting iteration 1
+Rank [8/16]: Start tile-wise ring communication...
+Rank [8/16]: Starting iteration 1
+Rank [15/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 2
+Rank [13/16]: Starting iteration 2
+Rank [12/16]: Starting iteration 2
+Rank [11/16]: Starting iteration 2
+Rank [10/16]: Starting iteration 2
+Rank [9/16]: Starting iteration 2
+Rank [8/16]: Starting iteration 2
+Rank [7/16]: Starting iteration 2
+Rank [6/16]: Starting iteration 2
+Rank [5/16]: Starting iteration 2
+Rank [4/16]: Starting iteration 2
+Rank [3/16]: Starting iteration 2
+Rank [2/16]: Starting iteration 2
+Rank [1/16]: Starting iteration 2
+Rank [0/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 3
+Rank [15/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 3
+Rank [12/16]: Starting iteration 3
+Rank [11/16]: Starting iteration 3
+Rank [10/16]: Starting iteration 3
+Rank [9/16]: Starting iteration 3
+Rank [8/16]: Starting iteration 3
+Rank [7/16]: Starting iteration 3
+Rank [6/16]: Starting iteration 3
+Rank [5/16]: Starting iteration 3
+Rank [4/16]: Starting iteration 3
+Rank [1/16]: Starting iteration 3
+Rank [2/16]: Starting iteration 3
+Rank [3/16]: Starting iteration 3
+Rank [0/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 4
+Rank [14/16]: Starting iteration 4
+Rank [15/16]: Starting iteration 4
+Rank [10/16]: Starting iteration 4
+Rank [11/16]: Starting iteration 4
+Rank [12/16]: Starting iteration 4
+Rank [7/16]: Starting iteration 4
+Rank [8/16]: Starting iteration 4
+Rank [9/16]: Starting iteration 4
+Rank [1/16]: Starting iteration 4
+Rank [4/16]: Starting iteration 4
+Rank [5/16]: Starting iteration 4
+Rank [2/16]: Starting iteration 4
+Rank [6/16]: Starting iteration 4
+Rank [0/16]: Starting iteration 4
+Rank [3/16]: Starting iteration 4
+Rank [13/16]: Starting iteration 5
+Rank [14/16]: Starting iteration 5
+Rank [9/16]: Starting iteration 5
+Rank [10/16]: Starting iteration 5
+Rank [12/16]: Starting iteration 5
+Rank [15/16]: Starting iteration 5
+Rank [1/16]: Starting iteration 5
+Rank [5/16]: Starting iteration 5
+Rank [8/16]: Starting iteration 5
+Rank [2/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 5
+Rank [6/16]: Starting iteration 5
+Rank [0/16]: Starting iteration 5
+Rank [4/16]: Starting iteration 5
+Rank [7/16]: Starting iteration 5
+Rank [3/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 6
+Rank [13/16]: Starting iteration 6
+Rank [15/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 6
+Rank [1/16]: Starting iteration 6
+Rank [2/16]: Starting iteration 6
+Rank [10/16]: Starting iteration 6
+Rank [7/16]: Starting iteration 6
+Rank [6/16]: Starting iteration 6
+Rank [5/16]: Starting iteration 6
+Rank [0/16]: Starting iteration 6
+Rank [8/16]: Starting iteration 6
+Rank [14/16]: Starting iteration 6
+Rank [3/16]: Starting iteration 6
+Rank [4/16]: Starting iteration 6
+Rank [9/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 7
+Rank [13/16]: Starting iteration 7
+Rank [0/16]: Starting iteration 7
+Rank [6/16]: Starting iteration 7
+Rank [5/16]: Starting iteration 7
+Rank [11/16]: Starting iteration 7
+Rank [15/16]: Starting iteration 7
+Rank [4/16]: Starting iteration 7
+Rank [1/16]: Starting iteration 7
+Rank [10/16]: Starting iteration 7
+Rank [7/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 7
+Rank [3/16]: Starting iteration 7
+Rank [9/16]: Starting iteration 7
+Rank [2/16]: Starting iteration 7
+Rank [8/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 8
+Rank [5/16]: Starting iteration 8
+Rank [12/16]: Starting iteration 8
+Rank [7/16]: Starting iteration 8
+Rank [0/16]: Starting iteration 8
+Rank [15/16]: Starting iteration 8
+Rank [2/16]: Starting iteration 8
+Rank [9/16]: Starting iteration 8
+Rank [1/16]: Starting iteration 8
+Rank [8/16]: Starting iteration 8
+Rank [10/16]: Starting iteration 8
+Rank [3/16]: Starting iteration 8
+Rank [4/16]: Starting iteration 8
+Rank [11/16]: Starting iteration 8
+Rank [13/16]: Starting iteration 8
+Rank [6/16]: Starting iteration 8
+Rank [15/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.9735625241883099 s
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Run 8
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks...
+Rank [2/16]: Calculate diagonal blocks...
+Rank [7/16]: Calculate diagonal blocks...
+Rank [11/16]: Calculate diagonal blocks...
+Rank [15/16]: Calculate diagonal blocks...
+Rank [9/16]: Calculate diagonal blocks...
+Rank [1/16]: Calculate diagonal blocks...
+Rank [4/16]: Calculate diagonal blocks...
+Rank [8/16]: Calculate diagonal blocks...
+Rank [3/16]: Calculate diagonal blocks...
+Rank [13/16]: Calculate diagonal blocks...
+Rank [6/16]: Calculate diagonal blocks...
+Rank [10/16]: Calculate diagonal blocks...
+Rank [5/16]: Calculate diagonal blocks...
+Rank [14/16]: Calculate diagonal blocks...
+Rank [12/16]: Calculate diagonal blocks...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [0/16]: Starting iteration 1
+Rank [11/16]: Start tile-wise ring communication...
+Rank [11/16]: Starting iteration 1
+Rank [7/16]: Start tile-wise ring communication...
+Rank [7/16]: Starting iteration 1
+Rank [9/16]: Start tile-wise ring communication...
+Rank [9/16]: Starting iteration 1
+Rank [10/16]: Start tile-wise ring communication...
+Rank [10/16]: Starting iteration 1
+Rank [5/16]: Start tile-wise ring communication...
+Rank [5/16]: Starting iteration 1
+Rank [2/16]: Start tile-wise ring communication...
+Rank [2/16]: Starting iteration 1
+Rank [15/16]: Start tile-wise ring communication...
+Rank [15/16]: Starting iteration 1
+Rank [4/16]: Start tile-wise ring communication...
+Rank [4/16]: Starting iteration 1
+Rank [8/16]: Start tile-wise ring communication...
+Rank [8/16]: Starting iteration 1
+Rank [12/16]: Start tile-wise ring communication...
+Rank [12/16]: Starting iteration 1
+Rank [14/16]: Start tile-wise ring communication...
+Rank [14/16]: Starting iteration 1
+Rank [1/16]: Start tile-wise ring communication...
+Rank [1/16]: Starting iteration 1
+Rank [3/16]: Start tile-wise ring communication...
+Rank [3/16]: Starting iteration 1
+Rank [13/16]: Start tile-wise ring communication...
+Rank [13/16]: Starting iteration 1
+Rank [6/16]: Start tile-wise ring communication...
+Rank [6/16]: Starting iteration 1
+Rank [15/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 2
+Rank [13/16]: Starting iteration 2
+Rank [12/16]: Starting iteration 2
+Rank [11/16]: Starting iteration 2
+Rank [10/16]: Starting iteration 2
+Rank [9/16]: Starting iteration 2
+Rank [8/16]: Starting iteration 2
+Rank [7/16]: Starting iteration 2
+Rank [6/16]: Starting iteration 2
+Rank [5/16]: Starting iteration 2
+Rank [4/16]: Starting iteration 2
+Rank [3/16]: Starting iteration 2
+Rank [2/16]: Starting iteration 2
+Rank [0/16]: Starting iteration 2
+Rank [1/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 3
+Rank [15/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 3
+Rank [12/16]: Starting iteration 3
+Rank [11/16]: Starting iteration 3
+Rank [10/16]: Starting iteration 3
+Rank [8/16]: Starting iteration 3
+Rank [9/16]: Starting iteration 3
+Rank [7/16]: Starting iteration 3
+Rank [6/16]: Starting iteration 3
+Rank [5/16]: Starting iteration 3
+Rank [4/16]: Starting iteration 3
+Rank [1/16]: Starting iteration 3
+Rank [0/16]: Starting iteration 3
+Rank [3/16]: Starting iteration 3
+Rank [2/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 4
+Rank [14/16]: Starting iteration 4
+Rank [15/16]: Starting iteration 4
+Rank [10/16]: Starting iteration 4
+Rank [11/16]: Starting iteration 4
+Rank [12/16]: Starting iteration 4
+Rank [7/16]: Starting iteration 4
+Rank [8/16]: Starting iteration 4
+Rank [9/16]: Starting iteration 4
+Rank [1/16]: Starting iteration 4
+Rank [4/16]: Starting iteration 4
+Rank [2/16]: Starting iteration 4
+Rank [5/16]: Starting iteration 4
+Rank [6/16]: Starting iteration 4
+Rank [0/16]: Starting iteration 4
+Rank [3/16]: Starting iteration 4
+Rank [13/16]: Starting iteration 5
+Rank [14/16]: Starting iteration 5
+Rank [12/16]: Starting iteration 5
+Rank [9/16]: Starting iteration 5
+Rank [10/16]: Starting iteration 5
+Rank [15/16]: Starting iteration 5
+Rank [1/16]: Starting iteration 5
+Rank [8/16]: Starting iteration 5
+Rank [5/16]: Starting iteration 5
+Rank [2/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 5
+Rank [6/16]: Starting iteration 5
+Rank [0/16]: Starting iteration 5
+Rank [4/16]: Starting iteration 5
+Rank [7/16]: Starting iteration 5
+Rank [3/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 6
+Rank [15/16]: Starting iteration 6
+Rank [13/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 6
+Rank [1/16]: Starting iteration 6
+Rank [10/16]: Starting iteration 6
+Rank [2/16]: Starting iteration 6
+Rank [6/16]: Starting iteration 6
+Rank [7/16]: Starting iteration 6
+Rank [0/16]: Starting iteration 6
+Rank [5/16]: Starting iteration 6
+Rank [14/16]: Starting iteration 6
+Rank [8/16]: Starting iteration 6
+Rank [3/16]: Starting iteration 6
+Rank [4/16]: Starting iteration 6
+Rank [9/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 7
+Rank [0/16]: Starting iteration 7
+Rank [5/16]: Starting iteration 7
+Rank [11/16]: Starting iteration 7
+Rank [6/16]: Starting iteration 7
+Rank [15/16]: Starting iteration 7
+Rank [13/16]: Starting iteration 7
+Rank [3/16]: Starting iteration 7
+Rank [9/16]: Starting iteration 7
+Rank [1/16]: Starting iteration 7
+Rank [7/16]: Starting iteration 7
+Rank [4/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 7
+Rank [10/16]: Starting iteration 7
+Rank [2/16]: Starting iteration 7
+Rank [8/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 8
+Rank [0/16]: Starting iteration 8
+Rank [7/16]: Starting iteration 8
+Rank [15/16]: Starting iteration 8
+Rank [5/16]: Starting iteration 8
+Rank [12/16]: Starting iteration 8
+Rank [13/16]: Starting iteration 8
+Rank [6/16]: Starting iteration 8
+Rank [1/16]: Starting iteration 8
+Rank [10/16]: Starting iteration 8
+Rank [8/16]: Starting iteration 8
+Rank [3/16]: Starting iteration 8
+Rank [2/16]: Starting iteration 8
+Rank [4/16]: Starting iteration 8
+Rank [11/16]: Starting iteration 8
+Rank [9/16]: Starting iteration 8
+Rank [15/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.9829646210419014 s
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Run 9
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks...
+Rank [4/16]: Calculate diagonal blocks...
+Rank [1/16]: Calculate diagonal blocks...
+Rank [11/16]: Calculate diagonal blocks...
+Rank [5/16]: Calculate diagonal blocks...
+Rank [2/16]: Calculate diagonal blocks...
+Rank [6/16]: Calculate diagonal blocks...
+Rank [14/16]: Calculate diagonal blocks...
+Rank [10/16]: Calculate diagonal blocks...
+Rank [3/16]: Calculate diagonal blocks...
+Rank [12/16]: Calculate diagonal blocks...
+Rank [15/16]: Calculate diagonal blocks...
+Rank [7/16]: Calculate diagonal blocks...
+Rank [8/16]: Calculate diagonal blocks...
+Rank [9/16]: Calculate diagonal blocks...
+Rank [13/16]: Calculate diagonal blocks...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [0/16]: Starting iteration 1
+Rank [15/16]: Start tile-wise ring communication...
+Rank [15/16]: Starting iteration 1
+Rank [7/16]: Start tile-wise ring communication...
+Rank [7/16]: Starting iteration 1
+Rank [11/16]: Start tile-wise ring communication...
+Rank [11/16]: Starting iteration 1
+Rank [12/16]: Start tile-wise ring communication...
+Rank [12/16]: Starting iteration 1
+Rank [9/16]: Start tile-wise ring communication...
+Rank [9/16]: Starting iteration 1
+Rank [10/16]: Start tile-wise ring communication...
+Rank [10/16]: Starting iteration 1
+Rank [1/16]: Start tile-wise ring communication...
+Rank [1/16]: Starting iteration 1
+Rank [6/16]: Start tile-wise ring communication...
+Rank [6/16]: Starting iteration 1
+Rank [8/16]: Start tile-wise ring communication...
+Rank [8/16]: Starting iteration 1
+Rank [13/16]: Start tile-wise ring communication...
+Rank [13/16]: Starting iteration 1
+Rank [4/16]: Start tile-wise ring communication...
+Rank [4/16]: Starting iteration 1
+Rank [3/16]: Start tile-wise ring communication...
+Rank [3/16]: Starting iteration 1
+Rank [5/16]: Start tile-wise ring communication...
+Rank [5/16]: Starting iteration 1
+Rank [2/16]: Start tile-wise ring communication...
+Rank [2/16]: Starting iteration 1
+Rank [14/16]: Start tile-wise ring communication...
+Rank [14/16]: Starting iteration 1
+Rank [15/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 2
+Rank [13/16]: Starting iteration 2
+Rank [12/16]: Starting iteration 2
+Rank [11/16]: Starting iteration 2
+Rank [10/16]: Starting iteration 2
+Rank [9/16]: Starting iteration 2
+Rank [8/16]: Starting iteration 2
+Rank [7/16]: Starting iteration 2
+Rank [6/16]: Starting iteration 2
+Rank [5/16]: Starting iteration 2
+Rank [4/16]: Starting iteration 2
+Rank [3/16]: Starting iteration 2
+Rank [2/16]: Starting iteration 2
+Rank [0/16]: Starting iteration 2
+Rank [1/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 3
+Rank [15/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 3
+Rank [12/16]: Starting iteration 3
+Rank [10/16]: Starting iteration 3
+Rank [11/16]: Starting iteration 3
+Rank [8/16]: Starting iteration 3
+Rank [9/16]: Starting iteration 3
+Rank [7/16]: Starting iteration 3
+Rank [6/16]: Starting iteration 3
+Rank [4/16]: Starting iteration 3
+Rank [5/16]: Starting iteration 3
+Rank [1/16]: Starting iteration 3
+Rank [2/16]: Starting iteration 3
+Rank [0/16]: Starting iteration 3
+Rank [3/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 4
+Rank [14/16]: Starting iteration 4
+Rank [15/16]: Starting iteration 4
+Rank [10/16]: Starting iteration 4
+Rank [11/16]: Starting iteration 4
+Rank [12/16]: Starting iteration 4
+Rank [7/16]: Starting iteration 4
+Rank [8/16]: Starting iteration 4
+Rank [9/16]: Starting iteration 4
+Rank [2/16]: Starting iteration 4
+Rank [4/16]: Starting iteration 4
+Rank [1/16]: Starting iteration 4
+Rank [5/16]: Starting iteration 4
+Rank [6/16]: Starting iteration 4
+Rank [0/16]: Starting iteration 4
+Rank [3/16]: Starting iteration 4
+Rank [13/16]: Starting iteration 5
+Rank [14/16]: Starting iteration 5
+Rank [9/16]: Starting iteration 5
+Rank [10/16]: Starting iteration 5
+Rank [12/16]: Starting iteration 5
+Rank [15/16]: Starting iteration 5
+Rank [1/16]: Starting iteration 5
+Rank [5/16]: Starting iteration 5
+Rank [2/16]: Starting iteration 5
+Rank [8/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 5
+Rank [6/16]: Starting iteration 5
+Rank [0/16]: Starting iteration 5
+Rank [4/16]: Starting iteration 5
+Rank [7/16]: Starting iteration 5
+Rank [3/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 6
+Rank [13/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 6
+Rank [15/16]: Starting iteration 6
+Rank [1/16]: Starting iteration 6
+Rank [2/16]: Starting iteration 6
+Rank [7/16]: Starting iteration 6
+Rank [6/16]: Starting iteration 6
+Rank [10/16]: Starting iteration 6
+Rank [5/16]: Starting iteration 6
+Rank [0/16]: Starting iteration 6
+Rank [14/16]: Starting iteration 6
+Rank [8/16]: Starting iteration 6
+Rank [3/16]: Starting iteration 6
+Rank [4/16]: Starting iteration 6
+Rank [9/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 7
+Rank [0/16]: Starting iteration 7
+Rank [13/16]: Starting iteration 7
+Rank [6/16]: Starting iteration 7
+Rank [5/16]: Starting iteration 7
+Rank [11/16]: Starting iteration 7
+Rank [15/16]: Starting iteration 7
+Rank [1/16]: Starting iteration 7
+Rank [4/16]: Starting iteration 7
+Rank [10/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 7
+Rank [7/16]: Starting iteration 7
+Rank [3/16]: Starting iteration 7
+Rank [9/16]: Starting iteration 7
+Rank [2/16]: Starting iteration 7
+Rank [8/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 8
+Rank [0/16]: Starting iteration 8
+Rank [7/16]: Starting iteration 8
+Rank [15/16]: Starting iteration 8
+Rank [5/16]: Starting iteration 8
+Rank [12/16]: Starting iteration 8
+Rank [13/16]: Starting iteration 8
+Rank [6/16]: Starting iteration 8
+Rank [1/16]: Starting iteration 8
+Rank [10/16]: Starting iteration 8
+Rank [8/16]: Starting iteration 8
+Rank [3/16]: Starting iteration 8
+Rank [4/16]: Starting iteration 8
+Rank [2/16]: Starting iteration 8
+Rank [11/16]: Starting iteration 8
+Rank [9/16]: Starting iteration 8
+Rank [15/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.9878637390211225 s
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Run 10
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks...
+Rank [9/16]: Calculate diagonal blocks...
+Rank [1/16]: Calculate diagonal blocks...
+Rank [12/16]: Calculate diagonal blocks...
+Rank [6/16]: Calculate diagonal blocks...
+Rank [5/16]: Calculate diagonal blocks...
+Rank [10/16]: Calculate diagonal blocks...
+Rank [13/16]: Calculate diagonal blocks...
+Rank [11/16]: Calculate diagonal blocks...
+Rank [14/16]: Calculate diagonal blocks...
+Rank [4/16]: Calculate diagonal blocks...
+Rank [7/16]: Calculate diagonal blocks...
+Rank [2/16]: Calculate diagonal blocks...
+Rank [8/16]: Calculate diagonal blocks...
+Rank [3/16]: Calculate diagonal blocks...
+Rank [15/16]: Calculate diagonal blocks...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [0/16]: Starting iteration 1
+Rank [9/16]: Start tile-wise ring communication...
+Rank [9/16]: Starting iteration 1
+Rank [7/16]: Start tile-wise ring communication...
+Rank [7/16]: Starting iteration 1
+Rank [2/16]: Start tile-wise ring communication...
+Rank [2/16]: Starting iteration 1
+Rank [11/16]: Start tile-wise ring communication...
+Rank [11/16]: Starting iteration 1
+Rank [14/16]: Start tile-wise ring communication...
+Rank [14/16]: Starting iteration 1
+Rank [1/16]: Start tile-wise ring communication...
+Rank [1/16]: Starting iteration 1
+Rank [4/16]: Start tile-wise ring communication...
+Rank [4/16]: Starting iteration 1
+Rank [8/16]: Start tile-wise ring communication...
+Rank [8/16]: Starting iteration 1
+Rank [3/16]: Start tile-wise ring communication...
+Rank [3/16]: Starting iteration 1
+Rank [6/16]: Start tile-wise ring communication...
+Rank [6/16]: Starting iteration 1
+Rank [5/16]: Start tile-wise ring communication...
+Rank [5/16]: Starting iteration 1
+Rank [10/16]: Start tile-wise ring communication...
+Rank [10/16]: Starting iteration 1
+Rank [13/16]: Start tile-wise ring communication...
+Rank [13/16]: Starting iteration 1
+Rank [15/16]: Start tile-wise ring communication...
+Rank [15/16]: Starting iteration 1
+Rank [12/16]: Start tile-wise ring communication...
+Rank [12/16]: Starting iteration 1
+Rank [15/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 2
+Rank [13/16]: Starting iteration 2
+Rank [12/16]: Starting iteration 2
+Rank [11/16]: Starting iteration 2
+Rank [10/16]: Starting iteration 2
+Rank [9/16]: Starting iteration 2
+Rank [8/16]: Starting iteration 2
+Rank [7/16]: Starting iteration 2
+Rank [6/16]: Starting iteration 2
+Rank [5/16]: Starting iteration 2
+Rank [4/16]: Starting iteration 2
+Rank [3/16]: Starting iteration 2
+Rank [2/16]: Starting iteration 2
+Rank [0/16]: Starting iteration 2
+Rank [1/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 3
+Rank [15/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 3
+Rank [12/16]: Starting iteration 3
+Rank [11/16]: Starting iteration 3
+Rank [10/16]: Starting iteration 3
+Rank [8/16]: Starting iteration 3
+Rank [9/16]: Starting iteration 3
+Rank [7/16]: Starting iteration 3
+Rank [6/16]: Starting iteration 3
+Rank [5/16]: Starting iteration 3
+Rank [4/16]: Starting iteration 3
+Rank [1/16]: Starting iteration 3
+Rank [0/16]: Starting iteration 3
+Rank [2/16]: Starting iteration 3
+Rank [3/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 4
+Rank [14/16]: Starting iteration 4
+Rank [15/16]: Starting iteration 4
+Rank [10/16]: Starting iteration 4
+Rank [11/16]: Starting iteration 4
+Rank [12/16]: Starting iteration 4
+Rank [7/16]: Starting iteration 4
+Rank [8/16]: Starting iteration 4
+Rank [9/16]: Starting iteration 4
+Rank [1/16]: Starting iteration 4
+Rank [4/16]: Starting iteration 4
+Rank [2/16]: Starting iteration 4
+Rank [5/16]: Starting iteration 4
+Rank [6/16]: Starting iteration 4
+Rank [0/16]: Starting iteration 4
+Rank [3/16]: Starting iteration 4
+Rank [13/16]: Starting iteration 5
+Rank [14/16]: Starting iteration 5
+Rank [9/16]: Starting iteration 5
+Rank [12/16]: Starting iteration 5
+Rank [10/16]: Starting iteration 5
+Rank [15/16]: Starting iteration 5
+Rank [1/16]: Starting iteration 5
+Rank [5/16]: Starting iteration 5
+Rank [8/16]: Starting iteration 5
+Rank [2/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 5
+Rank [6/16]: Starting iteration 5
+Rank [0/16]: Starting iteration 5
+Rank [4/16]: Starting iteration 5
+Rank [7/16]: Starting iteration 5
+Rank [3/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 6
+Rank [15/16]: Starting iteration 6
+Rank [13/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 6
+Rank [1/16]: Starting iteration 6
+Rank [10/16]: Starting iteration 6
+Rank [6/16]: Starting iteration 6
+Rank [2/16]: Starting iteration 6
+Rank [7/16]: Starting iteration 6
+Rank [0/16]: Starting iteration 6
+Rank [5/16]: Starting iteration 6
+Rank [8/16]: Starting iteration 6
+Rank [14/16]: Starting iteration 6
+Rank [3/16]: Starting iteration 6
+Rank [4/16]: Starting iteration 6
+Rank [9/16]: Starting iteration 6
+Rank [5/16]: Starting iteration 7
+Rank [11/16]: Starting iteration 7
+Rank [15/16]: Starting iteration 7
+Rank [12/16]: Starting iteration 7
+Rank [0/16]: Starting iteration 7
+Rank [13/16]: Starting iteration 7
+Rank [3/16]: Starting iteration 7
+Rank [6/16]: Starting iteration 7
+Rank [9/16]: Starting iteration 7
+Rank [4/16]: Starting iteration 7
+Rank [10/16]: Starting iteration 7
+Rank [1/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 7
+Rank [7/16]: Starting iteration 7
+Rank [2/16]: Starting iteration 7
+Rank [8/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 8
+Rank [0/16]: Starting iteration 8
+Rank [15/16]: Starting iteration 8
+Rank [7/16]: Starting iteration 8
+Rank [1/16]: Starting iteration 8
+Rank [8/16]: Starting iteration 8
+Rank [5/16]: Starting iteration 8
+Rank [10/16]: Starting iteration 8
+Rank [3/16]: Starting iteration 8
+Rank [12/16]: Starting iteration 8
+Rank [13/16]: Starting iteration 8
+Rank [6/16]: Starting iteration 8
+Rank [2/16]: Starting iteration 8
+Rank [4/16]: Starting iteration 8
+Rank [11/16]: Starting iteration 8
+Rank [9/16]: Starting iteration 8
+Rank [8/16]: [DONE]
+Rank [15/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.9931655330001377 s
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Run 11
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks...
+Rank [6/16]: Calculate diagonal blocks...
+Rank [14/16]: Calculate diagonal blocks...
+Rank [8/16]: Calculate diagonal blocks...
+Rank [13/16]: Calculate diagonal blocks...
+Rank [11/16]: Calculate diagonal blocks...
+Rank [1/16]: Calculate diagonal blocks...
+Rank [3/16]: Calculate diagonal blocks...
+Rank [12/16]: Calculate diagonal blocks...
+Rank [10/16]: Calculate diagonal blocks...
+Rank [4/16]: Calculate diagonal blocks...
+Rank [7/16]: Calculate diagonal blocks...
+Rank [9/16]: Calculate diagonal blocks...
+Rank [5/16]: Calculate diagonal blocks...
+Rank [15/16]: Calculate diagonal blocks...
+Rank [2/16]: Calculate diagonal blocks...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [0/16]: Starting iteration 1
+Rank [7/16]: Start tile-wise ring communication...
+Rank [7/16]: Starting iteration 1
+Rank [5/16]: Start tile-wise ring communication...
+Rank [5/16]: Starting iteration 1
+Rank [11/16]: Start tile-wise ring communication...
+Rank [11/16]: Starting iteration 1
+Rank [2/16]: Start tile-wise ring communication...
+Rank [2/16]: Starting iteration 1
+Rank [3/16]: Start tile-wise ring communication...
+Rank [3/16]: Starting iteration 1
+Rank [12/16]: Start tile-wise ring communication...
+Rank [12/16]: Starting iteration 1
+Rank [10/16]: Start tile-wise ring communication...
+Rank [10/16]: Starting iteration 1
+Rank [4/16]: Start tile-wise ring communication...
+Rank [4/16]: Starting iteration 1
+Rank [8/16]: Start tile-wise ring communication...
+Rank [8/16]: Starting iteration 1
+Rank [9/16]: Start tile-wise ring communication...
+Rank [9/16]: Starting iteration 1
+Rank [15/16]: Start tile-wise ring communication...
+Rank [15/16]: Starting iteration 1
+Rank [6/16]: Start tile-wise ring communication...
+Rank [6/16]: Starting iteration 1
+Rank [13/16]: Start tile-wise ring communication...
+Rank [13/16]: Starting iteration 1
+Rank [1/16]: Start tile-wise ring communication...
+Rank [1/16]: Starting iteration 1
+Rank [14/16]: Start tile-wise ring communication...
+Rank [14/16]: Starting iteration 1
+Rank [15/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 2
+Rank [13/16]: Starting iteration 2
+Rank [12/16]: Starting iteration 2
+Rank [11/16]: Starting iteration 2
+Rank [10/16]: Starting iteration 2
+Rank [9/16]: Starting iteration 2
+Rank [8/16]: Starting iteration 2
+Rank [7/16]: Starting iteration 2
+Rank [6/16]: Starting iteration 2
+Rank [5/16]: Starting iteration 2
+Rank [4/16]: Starting iteration 2
+Rank [3/16]: Starting iteration 2
+Rank [2/16]: Starting iteration 2
+Rank [0/16]: Starting iteration 2
+Rank [1/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 3
+Rank [15/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 3
+Rank [12/16]: Starting iteration 3
+Rank [11/16]: Starting iteration 3
+Rank [10/16]: Starting iteration 3
+Rank [8/16]: Starting iteration 3
+Rank [9/16]: Starting iteration 3
+Rank [7/16]: Starting iteration 3
+Rank [6/16]: Starting iteration 3
+Rank [5/16]: Starting iteration 3
+Rank [4/16]: Starting iteration 3
+Rank [0/16]: Starting iteration 3
+Rank [1/16]: Starting iteration 3
+Rank [2/16]: Starting iteration 3
+Rank [3/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 4
+Rank [14/16]: Starting iteration 4
+Rank [15/16]: Starting iteration 4
+Rank [10/16]: Starting iteration 4
+Rank [11/16]: Starting iteration 4
+Rank [12/16]: Starting iteration 4
+Rank [7/16]: Starting iteration 4
+Rank [8/16]: Starting iteration 4
+Rank [9/16]: Starting iteration 4
+Rank [2/16]: Starting iteration 4
+Rank [1/16]: Starting iteration 4
+Rank [4/16]: Starting iteration 4
+Rank [5/16]: Starting iteration 4
+Rank [6/16]: Starting iteration 4
+Rank [0/16]: Starting iteration 4
+Rank [3/16]: Starting iteration 4
+Rank [13/16]: Starting iteration 5
+Rank [14/16]: Starting iteration 5
+Rank [9/16]: Starting iteration 5
+Rank [12/16]: Starting iteration 5
+Rank [10/16]: Starting iteration 5
+Rank [15/16]: Starting iteration 5
+Rank [1/16]: Starting iteration 5
+Rank [5/16]: Starting iteration 5
+Rank [8/16]: Starting iteration 5
+Rank [2/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 5
+Rank [6/16]: Starting iteration 5
+Rank [0/16]: Starting iteration 5
+Rank [4/16]: Starting iteration 5
+Rank [7/16]: Starting iteration 5
+Rank [3/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 6
+Rank [15/16]: Starting iteration 6
+Rank [13/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 6
+Rank [1/16]: Starting iteration 6
+Rank [10/16]: Starting iteration 6
+Rank [2/16]: Starting iteration 6
+Rank [6/16]: Starting iteration 6
+Rank [7/16]: Starting iteration 6
+Rank [0/16]: Starting iteration 6
+Rank [5/16]: Starting iteration 6
+Rank [14/16]: Starting iteration 6
+Rank [8/16]: Starting iteration 6
+Rank [3/16]: Starting iteration 6
+Rank [4/16]: Starting iteration 6
+Rank [9/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 7
+Rank [13/16]: Starting iteration 7
+Rank [0/16]: Starting iteration 7
+Rank [6/16]: Starting iteration 7
+Rank [5/16]: Starting iteration 7
+Rank [11/16]: Starting iteration 7
+Rank [15/16]: Starting iteration 7
+Rank [4/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 7
+Rank [10/16]: Starting iteration 7
+Rank [1/16]: Starting iteration 7
+Rank [7/16]: Starting iteration 7
+Rank [3/16]: Starting iteration 7
+Rank [9/16]: Starting iteration 7
+Rank [2/16]: Starting iteration 7
+Rank [8/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 8
+Rank [15/16]: Starting iteration 8
+Rank [0/16]: Starting iteration 8
+Rank [7/16]: Starting iteration 8
+Rank [13/16]: Starting iteration 8
+Rank [6/16]: Starting iteration 8
+Rank [5/16]: Starting iteration 8
+Rank [12/16]: Starting iteration 8
+Rank [1/16]: Starting iteration 8
+Rank [8/16]: Starting iteration 8
+Rank [10/16]: Starting iteration 8
+Rank [3/16]: Starting iteration 8
+Rank [2/16]: Starting iteration 8
+Rank [4/16]: Starting iteration 8
+Rank [11/16]: Starting iteration 8
+Rank [9/16]: Starting iteration 8
+Rank [15/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.9847494116984308 s
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+
+============================= JOB FEEDBACK =============================
+
+NodeName=uc2n[131-132,152,154,186-190,204-205,208,215-217,220]
+Job ID: 24894113
+Cluster: uc2
+User/Group: ku4408/scc
+State: COMPLETED (exit code 0)
+Nodes: 16
+Cores per node: 80
+CPU Utilized: 00:09:21
+CPU Efficiency: 0.43% of 1-12:37:20 core-walltime
+Job Wall-clock time: 00:01:43
+Memory Utilized: 1.14 GB
+Memory Efficiency: 0.08% of 1.37 TB
diff --git a/2_psrs_cdist/solutions/A1/strong_scaling_symmetric/nodes_2.out b/2_psrs_cdist/solutions/A1/strong_scaling_symmetric/nodes_2.out
new file mode 100644
index 0000000..7eded11
--- /dev/null
+++ b/2_psrs_cdist/solutions/A1/strong_scaling_symmetric/nodes_2.out
@@ -0,0 +1,279 @@
+Run 1
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/2]: Local data chunk has shape [25000, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/2]: Local data chunk has shape [25000, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/2]: Calculate diagonal blocks...
+Rank [1/2]: Calculate diagonal blocks...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [0/2]: Starting iteration 1
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: Starting iteration 1
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [25000, 50000].
+Process-averaged run time: 6.266019005328417 s
+Rank [1/2]: Local distance matrix has shape [25000, 50000].
+Run 2
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [1/2]: Local data chunk has shape [25000, 18]...
+ [OK]
+Rank [0/2]: Local data chunk has shape [25000, 18]...
+Start distance calculations...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/2]: Calculate diagonal blocks...
+Rank [1/2]: Calculate diagonal blocks...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [0/2]: Starting iteration 1
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: Starting iteration 1
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [25000, 50000].
+Process-averaged run time: 6.181757535319775 s
+Rank [1/2]: Local distance matrix has shape [25000, 50000].
+Run 3
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [1/2]: Local data chunk has shape [25000, 18]...
+ [OK]
+Rank [0/2]: Local data chunk has shape [25000, 18]...
+Start distance calculations...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/2]: Calculate diagonal blocks...
+Rank [1/2]: Calculate diagonal blocks...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [0/2]: Starting iteration 1
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: Starting iteration 1
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [25000, 50000].
+Process-averaged run time: 6.169249661266804 s
+Rank [1/2]: Local distance matrix has shape [25000, 50000].
+Run 4
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/2]: Local data chunk has shape [25000, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/2]: Local data chunk has shape [25000, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/2]: Calculate diagonal blocks...
+Rank [1/2]: Calculate diagonal blocks...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [0/2]: Starting iteration 1
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: Starting iteration 1
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [25000, 50000].
+Process-averaged run time: 6.169275370426476 s
+Rank [1/2]: Local distance matrix has shape [25000, 50000].
+Run 5
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/2]: Local data chunk has shape [25000, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/2]: Local data chunk has shape [25000, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/2]: Calculate diagonal blocks...
+Rank [1/2]: Calculate diagonal blocks...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [0/2]: Starting iteration 1
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: Starting iteration 1
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [25000, 50000].
+Process-averaged run time: 6.171142612583935 s
+Rank [1/2]: Local distance matrix has shape [25000, 50000].
+Run 6
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/2]: Local data chunk has shape [25000, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/2]: Local data chunk has shape [25000, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/2]: Calculate diagonal blocks...
+Rank [1/2]: Calculate diagonal blocks...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [0/2]: Starting iteration 1
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: Starting iteration 1
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [25000, 50000].
+Process-averaged run time: 6.167563481256366 s
+Rank [1/2]: Local distance matrix has shape [25000, 50000].
+Run 7
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [1/2]: Local data chunk has shape [25000, 18]...
+ [OK]
+Rank [0/2]: Local data chunk has shape [25000, 18]...
+Start distance calculations...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/2]: Calculate diagonal blocks...
+Rank [1/2]: Calculate diagonal blocks...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [0/2]: Starting iteration 1
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: Starting iteration 1
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [25000, 50000].
+Process-averaged run time: 6.172976263333112 s
+Rank [1/2]: Local distance matrix has shape [25000, 50000].
+Run 8
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/2]: Local data chunk has shape [25000, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/2]: Local data chunk has shape [25000, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/2]: Calculate diagonal blocks...
+Rank [1/2]: Calculate diagonal blocks...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [0/2]: Starting iteration 1
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: Starting iteration 1
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [25000, 50000].
+Process-averaged run time: 6.169111441355199 s
+Rank [1/2]: Local distance matrix has shape [25000, 50000].
+Run 9
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/2]: Local data chunk has shape [25000, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/2]: Local data chunk has shape [25000, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/2]: Calculate diagonal blocks...
+Rank [1/2]: Calculate diagonal blocks...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [0/2]: Starting iteration 1
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: Starting iteration 1
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [25000, 50000].
+Process-averaged run time: 6.173400138504803 s
+Rank [1/2]: Local distance matrix has shape [25000, 50000].
+Run 10
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/2]: Local data chunk has shape [25000, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/2]: Local data chunk has shape [25000, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/2]: Calculate diagonal blocks...
+Rank [1/2]: Calculate diagonal blocks...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [0/2]: Starting iteration 1
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: Starting iteration 1
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [25000, 50000].
+Process-averaged run time: 6.16715893195942 s
+Rank [1/2]: Local distance matrix has shape [25000, 50000].
+Run 11
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/2]: Local data chunk has shape [25000, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/2]: Local data chunk has shape [25000, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/2]: Calculate diagonal blocks...
+Rank [1/2]: Calculate diagonal blocks...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [0/2]: Starting iteration 1
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: Starting iteration 1
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [25000, 50000].
+Process-averaged run time: 6.168558022007346 s
+Rank [1/2]: Local distance matrix has shape [25000, 50000].
+
+============================= JOB FEEDBACK =============================
+
+NodeName=uc2n[005-006]
+Job ID: 24894109
+Cluster: uc2
+User/Group: ku4408/scc
+State: COMPLETED (exit code 0)
+Nodes: 2
+Cores per node: 80
+CPU Utilized: 00:03:21
+CPU Efficiency: 0.75% of 07:25:20 core-walltime
+Job Wall-clock time: 00:02:47
+Memory Utilized: 9.80 GB
+Memory Efficiency: 5.58% of 175.78 GB
diff --git a/2_psrs_cdist/solutions/A1/strong_scaling_symmetric/nodes_4.out b/2_psrs_cdist/solutions/A1/strong_scaling_symmetric/nodes_4.out
new file mode 100644
index 0000000..c0d40e6
--- /dev/null
+++ b/2_psrs_cdist/solutions/A1/strong_scaling_symmetric/nodes_4.out
@@ -0,0 +1,499 @@
+Run 1
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [3/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [1/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [2/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [0/4]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [3/4]: Calculate diagonal blocks...
+Rank [1/4]: Calculate diagonal blocks...
+Rank [0/4]: Calculate diagonal blocks...
+Rank [2/4]: Calculate diagonal blocks...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [0/4]: Starting iteration 1
+Rank [2/4]: Start tile-wise ring communication...
+Rank [2/4]: Starting iteration 1
+Rank [3/4]: Start tile-wise ring communication...
+Rank [3/4]: Starting iteration 1
+Rank [1/4]: Start tile-wise ring communication...
+Rank [1/4]: Starting iteration 1
+Rank [3/4]: Starting iteration 2
+Rank [2/4]: Starting iteration 2
+Rank [0/4]: Starting iteration 2
+Rank [1/4]: Starting iteration 2
+Rank [2/4]: [DONE]
+Rank [3/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [12500, 50000].
+Process-averaged run time: 3.0001420916523784 s
+Rank [1/4]: Local distance matrix has shape [12500, 50000].
+Rank [3/4]: Local distance matrix has shape [12500, 50000].
+Rank [2/4]: Local distance matrix has shape [12500, 50000].
+Run 2
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/4]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [2/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [3/4]: Local data chunk has shape [12500, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/4]: Calculate diagonal blocks...
+Rank [1/4]: Calculate diagonal blocks...
+Rank [3/4]: Calculate diagonal blocks...
+Rank [2/4]: Calculate diagonal blocks...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [0/4]: Starting iteration 1
+Rank [2/4]: Start tile-wise ring communication...
+Rank [2/4]: Starting iteration 1
+Rank [1/4]: Start tile-wise ring communication...
+Rank [1/4]: Starting iteration 1
+Rank [3/4]: Start tile-wise ring communication...
+Rank [3/4]: Starting iteration 1
+Rank [3/4]: Starting iteration 2
+Rank [2/4]: Starting iteration 2
+Rank [0/4]: Starting iteration 2
+Rank [1/4]: Starting iteration 2
+Rank [2/4]: [DONE]
+Rank [3/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [12500, 50000].
+Process-averaged run time: 2.891579987015575 s
+Rank [1/4]: Local distance matrix has shape [12500, 50000].
+Rank [3/4]: Local distance matrix has shape [12500, 50000].
+Rank [2/4]: Local distance matrix has shape [12500, 50000].
+Run 3
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [1/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [3/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [2/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [0/4]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/4]: Calculate diagonal blocks...
+Rank [1/4]: Calculate diagonal blocks...
+Rank [2/4]: Calculate diagonal blocks...
+Rank [3/4]: Calculate diagonal blocks...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [0/4]: Starting iteration 1
+Rank [2/4]: Start tile-wise ring communication...
+Rank [2/4]: Starting iteration 1
+Rank [1/4]: Start tile-wise ring communication...
+Rank [1/4]: Starting iteration 1
+Rank [3/4]: Start tile-wise ring communication...
+Rank [3/4]: Starting iteration 1
+Rank [3/4]: Starting iteration 2
+Rank [2/4]: Starting iteration 2
+Rank [0/4]: Starting iteration 2
+Rank [1/4]: Starting iteration 2
+Rank [2/4]: [DONE]
+Rank [3/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [12500, 50000].
+Process-averaged run time: 2.8908693362027407 s
+Rank [2/4]: Local distance matrix has shape [12500, 50000].
+Rank [3/4]: Local distance matrix has shape [12500, 50000].
+Rank [1/4]: Local distance matrix has shape [12500, 50000].
+Run 4
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/4]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+ [OK]
+Rank [3/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [1/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [2/4]: Local data chunk has shape [12500, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/4]: Calculate diagonal blocks...
+Rank [1/4]: Calculate diagonal blocks...
+Rank [2/4]: Calculate diagonal blocks...
+Rank [3/4]: Calculate diagonal blocks...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [0/4]: Starting iteration 1
+Rank [2/4]: Start tile-wise ring communication...
+Rank [2/4]: Starting iteration 1
+Rank [3/4]: Start tile-wise ring communication...
+Rank [3/4]: Starting iteration 1
+Rank [1/4]: Start tile-wise ring communication...
+Rank [1/4]: Starting iteration 1
+Rank [3/4]: Starting iteration 2
+Rank [2/4]: Starting iteration 2
+Rank [0/4]: Starting iteration 2
+Rank [1/4]: Starting iteration 2
+Rank [2/4]: [DONE]
+Rank [3/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [12500, 50000].
+Process-averaged run time: 2.8817238493356854 s
+Rank [2/4]: Local distance matrix has shape [12500, 50000].
+Rank [3/4]: Local distance matrix has shape [12500, 50000].
+Rank [1/4]: Local distance matrix has shape [12500, 50000].
+Run 5
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/4]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [3/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [2/4]: Local data chunk has shape [12500, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/4]: Calculate diagonal blocks...
+Rank [1/4]: Calculate diagonal blocks...
+Rank [2/4]: Calculate diagonal blocks...
+Rank [3/4]: Calculate diagonal blocks...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [0/4]: Starting iteration 1
+Rank [1/4]: Start tile-wise ring communication...
+Rank [1/4]: Starting iteration 1
+Rank [2/4]: Start tile-wise ring communication...
+Rank [2/4]: Starting iteration 1
+Rank [3/4]: Start tile-wise ring communication...
+Rank [3/4]: Starting iteration 1
+Rank [3/4]: Starting iteration 2
+Rank [2/4]: Starting iteration 2
+Rank [0/4]: Starting iteration 2
+Rank [1/4]: Starting iteration 2
+Rank [2/4]: [DONE]
+Rank [3/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [12500, 50000].
+Process-averaged run time: 2.8827228804584593 s
+Rank [2/4]: Local distance matrix has shape [12500, 50000].
+Rank [1/4]: Local distance matrix has shape [12500, 50000].
+Rank [3/4]: Local distance matrix has shape [12500, 50000].
+Run 6
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/4]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+ [OK]
+Rank [3/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [2/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [1/4]: Local data chunk has shape [12500, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/4]: Calculate diagonal blocks...
+Rank [2/4]: Calculate diagonal blocks...
+Rank [3/4]: Calculate diagonal blocks...
+Rank [1/4]: Calculate diagonal blocks...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [0/4]: Starting iteration 1
+Rank [2/4]: Start tile-wise ring communication...
+Rank [2/4]: Starting iteration 1
+Rank [3/4]: Start tile-wise ring communication...
+Rank [3/4]: Starting iteration 1
+Rank [1/4]: Start tile-wise ring communication...
+Rank [1/4]: Starting iteration 1
+Rank [3/4]: Starting iteration 2
+Rank [2/4]: Starting iteration 2
+Rank [0/4]: Starting iteration 2
+Rank [1/4]: Starting iteration 2
+Rank [2/4]: [DONE]
+Rank [3/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [12500, 50000].
+Process-averaged run time: 2.8919111364521086 s
+Rank [1/4]: Local distance matrix has shape [12500, 50000].
+Rank [2/4]: Local distance matrix has shape [12500, 50000].
+Rank [3/4]: Local distance matrix has shape [12500, 50000].
+Run 7
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/4]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+ [OK]
+Rank [2/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [3/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [1/4]: Local data chunk has shape [12500, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/4]: Calculate diagonal blocks...
+Rank [1/4]: Calculate diagonal blocks...
+Rank [2/4]: Calculate diagonal blocks...
+Rank [3/4]: Calculate diagonal blocks...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [0/4]: Starting iteration 1
+Rank [1/4]: Start tile-wise ring communication...
+Rank [1/4]: Starting iteration 1
+Rank [2/4]: Start tile-wise ring communication...
+Rank [2/4]: Starting iteration 1
+Rank [3/4]: Start tile-wise ring communication...
+Rank [3/4]: Starting iteration 1
+Rank [3/4]: Starting iteration 2
+Rank [2/4]: Starting iteration 2
+Rank [0/4]: Starting iteration 2
+Rank [1/4]: Starting iteration 2
+Rank [2/4]: [DONE]
+Rank [3/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [12500, 50000].
+Process-averaged run time: 2.8823061871808022 s
+Rank [2/4]: Local distance matrix has shape [12500, 50000].
+Rank [3/4]: Local distance matrix has shape [12500, 50000].
+Rank [1/4]: Local distance matrix has shape [12500, 50000].
+Run 8
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/4]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+ [OK]
+Rank [3/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [1/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [2/4]: Local data chunk has shape [12500, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/4]: Calculate diagonal blocks...
+Rank [1/4]: Calculate diagonal blocks...
+Rank [2/4]: Calculate diagonal blocks...
+Rank [3/4]: Calculate diagonal blocks...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [0/4]: Starting iteration 1
+Rank [1/4]: Start tile-wise ring communication...
+Rank [1/4]: Starting iteration 1
+Rank [2/4]: Start tile-wise ring communication...
+Rank [2/4]: Starting iteration 1
+Rank [3/4]: Start tile-wise ring communication...
+Rank [3/4]: Starting iteration 1
+Rank [3/4]: Starting iteration 2
+Rank [2/4]: Starting iteration 2
+Rank [0/4]: Starting iteration 2
+Rank [1/4]: Starting iteration 2
+Rank [2/4]: [DONE]
+Rank [3/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [12500, 50000].
+Process-averaged run time: 2.882331378525123 s
+Rank [1/4]: Local distance matrix has shape [12500, 50000].
+Rank [3/4]: Local distance matrix has shape [12500, 50000].
+Rank [2/4]: Local distance matrix has shape [12500, 50000].
+Run 9
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [2/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [3/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [1/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [0/4]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/4]: Calculate diagonal blocks...
+Rank [2/4]: Calculate diagonal blocks...
+Rank [1/4]: Calculate diagonal blocks...
+Rank [3/4]: Calculate diagonal blocks...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [0/4]: Starting iteration 1
+Rank [2/4]: Start tile-wise ring communication...
+Rank [2/4]: Starting iteration 1
+Rank [1/4]: Start tile-wise ring communication...
+Rank [1/4]: Starting iteration 1
+Rank [3/4]: Start tile-wise ring communication...
+Rank [3/4]: Starting iteration 1
+Rank [3/4]: Starting iteration 2
+Rank [2/4]: Starting iteration 2
+Rank [0/4]: Starting iteration 2
+Rank [1/4]: Starting iteration 2
+Rank [2/4]: [DONE]
+Rank [3/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [12500, 50000].
+Process-averaged run time: 2.8921143126208335 s
+Rank [2/4]: Local distance matrix has shape [12500, 50000].
+Rank [1/4]: Local distance matrix has shape [12500, 50000].
+Rank [3/4]: Local distance matrix has shape [12500, 50000].
+Run 10
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [1/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [3/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [2/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [0/4]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/4]: Calculate diagonal blocks...
+Rank [1/4]: Calculate diagonal blocks...
+Rank [2/4]: Calculate diagonal blocks...
+Rank [3/4]: Calculate diagonal blocks...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [0/4]: Starting iteration 1
+Rank [3/4]: Start tile-wise ring communication...
+Rank [3/4]: Starting iteration 1
+Rank [2/4]: Start tile-wise ring communication...
+Rank [2/4]: Starting iteration 1
+Rank [1/4]: Start tile-wise ring communication...
+Rank [1/4]: Starting iteration 1
+Rank [3/4]: Starting iteration 2
+Rank [2/4]: Starting iteration 2
+Rank [0/4]: Starting iteration 2
+Rank [1/4]: Starting iteration 2
+Rank [2/4]: [DONE]
+Rank [3/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [12500, 50000].
+Process-averaged run time: 2.8922928040847182 s
+Rank [2/4]: Local distance matrix has shape [12500, 50000].
+Rank [1/4]: Local distance matrix has shape [12500, 50000].
+Rank [3/4]: Local distance matrix has shape [12500, 50000].
+Run 11
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/4]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+ [OK]
+Rank [2/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [3/4]: Local data chunk has shape [12500, 18]...
+ [OK]
+Rank [1/4]: Local data chunk has shape [12500, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/4]: Calculate diagonal blocks...
+Rank [2/4]: Calculate diagonal blocks...
+Rank [1/4]: Calculate diagonal blocks...
+Rank [3/4]: Calculate diagonal blocks...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [0/4]: Starting iteration 1
+Rank [2/4]: Start tile-wise ring communication...
+Rank [2/4]: Starting iteration 1
+Rank [1/4]: Start tile-wise ring communication...
+Rank [1/4]: Starting iteration 1
+Rank [3/4]: Start tile-wise ring communication...
+Rank [3/4]: Starting iteration 1
+Rank [3/4]: Starting iteration 2
+Rank [2/4]: Starting iteration 2
+Rank [1/4]: Starting iteration 2
+Rank [0/4]: Starting iteration 2
+Rank [2/4]: [DONE]
+Rank [3/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [12500, 50000].
+Process-averaged run time: 2.8935788962990046 s
+Rank [1/4]: Local distance matrix has shape [12500, 50000].
+Rank [2/4]: Local distance matrix has shape [12500, 50000].
+Rank [3/4]: Local distance matrix has shape [12500, 50000].
+
+============================= JOB FEEDBACK =============================
+
+NodeName=uc2n[001-004]
+Job ID: 24894111
+Cluster: uc2
+User/Group: ku4408/scc
+State: COMPLETED (exit code 0)
+Nodes: 4
+Cores per node: 80
+CPU Utilized: 00:03:46
+CPU Efficiency: 0.57% of 11:01:20 core-walltime
+Job Wall-clock time: 00:02:04
+Memory Utilized: 3.97 GB
+Memory Efficiency: 1.13% of 351.56 GB
diff --git a/2_psrs_cdist/solutions/A1/strong_scaling_symmetric/nodes_8.out b/2_psrs_cdist/solutions/A1/strong_scaling_symmetric/nodes_8.out
new file mode 100644
index 0000000..faef493
--- /dev/null
+++ b/2_psrs_cdist/solutions/A1/strong_scaling_symmetric/nodes_8.out
@@ -0,0 +1,1071 @@
+Run 1
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [3/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [4/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [0/8]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [5/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [4/8]: Calculate diagonal blocks...
+Rank [3/8]: Calculate diagonal blocks...
+Rank [5/8]: Calculate diagonal blocks...
+Rank [6/8]: Calculate diagonal blocks...
+Rank [2/8]: Calculate diagonal blocks...
+Rank [7/8]: Calculate diagonal blocks...
+Rank [0/8]: Calculate diagonal blocks...
+Rank [1/8]: Calculate diagonal blocks...
+Rank [4/8]: Start tile-wise ring communication...
+Rank [4/8]: Starting iteration 1
+Rank [0/8]: Start tile-wise ring communication...
+Rank [0/8]: Starting iteration 1
+Rank [3/8]: Start tile-wise ring communication...
+Rank [3/8]: Starting iteration 1
+Rank [5/8]: Start tile-wise ring communication...
+Rank [5/8]: Starting iteration 1
+Rank [2/8]: Start tile-wise ring communication...
+Rank [2/8]: Starting iteration 1
+Rank [6/8]: Start tile-wise ring communication...
+Rank [6/8]: Starting iteration 1
+Rank [1/8]: Start tile-wise ring communication...
+Rank [1/8]: Starting iteration 1
+Rank [7/8]: Start tile-wise ring communication...
+Rank [7/8]: Starting iteration 1
+Rank [7/8]: Starting iteration 2
+Rank [6/8]: Starting iteration 2
+Rank [5/8]: Starting iteration 2
+Rank [4/8]: Starting iteration 2
+Rank [3/8]: Starting iteration 2
+Rank [2/8]: Starting iteration 2
+Rank [0/8]: Starting iteration 2
+Rank [1/8]: Starting iteration 2
+Rank [6/8]: Starting iteration 3
+Rank [7/8]: Starting iteration 3
+Rank [4/8]: Starting iteration 3
+Rank [5/8]: Starting iteration 3
+Rank [0/8]: Starting iteration 3
+Rank [2/8]: Starting iteration 3
+Rank [3/8]: Starting iteration 3
+Rank [1/8]: Starting iteration 3
+Rank [2/8]: Starting iteration 4
+Rank [7/8]: Starting iteration 4
+Rank [5/8]: Starting iteration 4
+Rank [4/8]: Starting iteration 4
+Rank [6/8]: Starting iteration 4
+Rank [1/8]: Starting iteration 4
+Rank [0/8]: Starting iteration 4
+Rank [3/8]: Starting iteration 4
+Rank [6/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [7/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [6250, 50000].
+Process-averaged run time: 1.7527669108239934 s
+Rank [3/8]: Local distance matrix has shape [6250, 50000].
+Rank [7/8]: Local distance matrix has shape [6250, 50000].
+Rank [2/8]: Local distance matrix has shape [6250, 50000].
+Rank [1/8]: Local distance matrix has shape [6250, 50000].
+Rank [5/8]: Local distance matrix has shape [6250, 50000].
+Rank [4/8]: Local distance matrix has shape [6250, 50000].
+Rank [6/8]: Local distance matrix has shape [6250, 50000].
+Run 2
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/8]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [7/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [4/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [5/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [3/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/8]: Calculate diagonal blocks...
+Rank [2/8]: Calculate diagonal blocks...
+Rank [3/8]: Calculate diagonal blocks...
+Rank [4/8]: Calculate diagonal blocks...
+Rank [1/8]: Calculate diagonal blocks...
+Rank [5/8]: Calculate diagonal blocks...
+Rank [6/8]: Calculate diagonal blocks...
+Rank [7/8]: Calculate diagonal blocks...
+Rank [0/8]: Start tile-wise ring communication...
+Rank [0/8]: Starting iteration 1
+Rank [2/8]: Start tile-wise ring communication...
+Rank [2/8]: Starting iteration 1
+Rank [4/8]: Start tile-wise ring communication...
+Rank [4/8]: Starting iteration 1
+Rank [1/8]: Start tile-wise ring communication...
+Rank [1/8]: Starting iteration 1
+Rank [3/8]: Start tile-wise ring communication...
+Rank [3/8]: Starting iteration 1
+Rank [6/8]: Start tile-wise ring communication...
+Rank [6/8]: Starting iteration 1
+Rank [7/8]: Start tile-wise ring communication...
+Rank [7/8]: Starting iteration 1
+Rank [5/8]: Start tile-wise ring communication...
+Rank [5/8]: Starting iteration 1
+Rank [7/8]: Starting iteration 2
+Rank [6/8]: Starting iteration 2
+Rank [5/8]: Starting iteration 2
+Rank [4/8]: Starting iteration 2
+Rank [3/8]: Starting iteration 2
+Rank [2/8]: Starting iteration 2
+Rank [0/8]: Starting iteration 2
+Rank [1/8]: Starting iteration 2
+Rank [6/8]: Starting iteration 3
+Rank [7/8]: Starting iteration 3
+Rank [4/8]: Starting iteration 3
+Rank [5/8]: Starting iteration 3
+Rank [0/8]: Starting iteration 3
+Rank [2/8]: Starting iteration 3
+Rank [3/8]: Starting iteration 3
+Rank [1/8]: Starting iteration 3
+Rank [2/8]: Starting iteration 4
+Rank [7/8]: Starting iteration 4
+Rank [5/8]: Starting iteration 4
+Rank [6/8]: Starting iteration 4
+Rank [4/8]: Starting iteration 4
+Rank [1/8]: Starting iteration 4
+Rank [0/8]: Starting iteration 4
+Rank [3/8]: Starting iteration 4
+Rank [6/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [7/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [6250, 50000].
+Process-averaged run time: 1.657699539558962 s
+Rank [1/8]: Local distance matrix has shape [6250, 50000].
+Rank [3/8]: Local distance matrix has shape [6250, 50000].
+Rank [6/8]: Local distance matrix has shape [6250, 50000].
+Rank [5/8]: Local distance matrix has shape [6250, 50000].
+Rank [7/8]: Local distance matrix has shape [6250, 50000].
+Rank [2/8]: Local distance matrix has shape [6250, 50000].
+Rank [4/8]: Local distance matrix has shape [6250, 50000].
+Run 3
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [5/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [3/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [4/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [0/8]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [7/8]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/8]: Calculate diagonal blocks...
+Rank [6/8]: Calculate diagonal blocks...
+Rank [2/8]: Calculate diagonal blocks...
+Rank [3/8]: Calculate diagonal blocks...
+Rank [5/8]: Calculate diagonal blocks...
+Rank [4/8]: Calculate diagonal blocks...
+Rank [1/8]: Calculate diagonal blocks...
+Rank [7/8]: Calculate diagonal blocks...
+Rank [0/8]: Start tile-wise ring communication...
+Rank [0/8]: Starting iteration 1
+Rank [4/8]: Start tile-wise ring communication...
+Rank [4/8]: Starting iteration 1
+Rank [6/8]: Start tile-wise ring communication...
+Rank [6/8]: Starting iteration 1
+Rank [2/8]: Start tile-wise ring communication...
+Rank [2/8]: Starting iteration 1
+Rank [3/8]: Start tile-wise ring communication...
+Rank [3/8]: Starting iteration 1
+Rank [1/8]: Start tile-wise ring communication...
+Rank [1/8]: Starting iteration 1
+Rank [5/8]: Start tile-wise ring communication...
+Rank [5/8]: Starting iteration 1
+Rank [7/8]: Start tile-wise ring communication...
+Rank [7/8]: Starting iteration 1
+Rank [7/8]: Starting iteration 2
+Rank [6/8]: Starting iteration 2
+Rank [5/8]: Starting iteration 2
+Rank [4/8]: Starting iteration 2
+Rank [3/8]: Starting iteration 2
+Rank [2/8]: Starting iteration 2
+Rank [0/8]: Starting iteration 2
+Rank [1/8]: Starting iteration 2
+Rank [6/8]: Starting iteration 3
+Rank [7/8]: Starting iteration 3
+Rank [4/8]: Starting iteration 3
+Rank [5/8]: Starting iteration 3
+Rank [0/8]: Starting iteration 3
+Rank [2/8]: Starting iteration 3
+Rank [3/8]: Starting iteration 3
+Rank [1/8]: Starting iteration 3
+Rank [7/8]: Starting iteration 4
+Rank [2/8]: Starting iteration 4
+Rank [5/8]: Starting iteration 4
+Rank [6/8]: Starting iteration 4
+Rank [4/8]: Starting iteration 4
+Rank [1/8]: Starting iteration 4
+Rank [0/8]: Starting iteration 4
+Rank [3/8]: Starting iteration 4
+Rank [6/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [7/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [6250, 50000].
+Process-averaged run time: 1.6521431785076857 s
+Rank [3/8]: Local distance matrix has shape [6250, 50000].
+Rank [5/8]: Local distance matrix has shape [6250, 50000].
+Rank [2/8]: Local distance matrix has shape [6250, 50000].
+Rank [4/8]: Local distance matrix has shape [6250, 50000].
+Rank [1/8]: Local distance matrix has shape [6250, 50000].
+Rank [6/8]: Local distance matrix has shape [6250, 50000].
+Rank [7/8]: Local distance matrix has shape [6250, 50000].
+Run 4
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/8]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [7/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [4/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [5/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [3/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/8]: Calculate diagonal blocks...
+Rank [2/8]: Calculate diagonal blocks...
+Rank [6/8]: Calculate diagonal blocks...
+Rank [7/8]: Calculate diagonal blocks...
+Rank [4/8]: Calculate diagonal blocks...
+Rank [5/8]: Calculate diagonal blocks...
+Rank [3/8]: Calculate diagonal blocks...
+Rank [1/8]: Calculate diagonal blocks...
+Rank [0/8]: Start tile-wise ring communication...
+Rank [0/8]: Starting iteration 1
+Rank [1/8]: Start tile-wise ring communication...
+Rank [1/8]: Starting iteration 1
+Rank [2/8]: Start tile-wise ring communication...
+Rank [2/8]: Starting iteration 1
+Rank [3/8]: Start tile-wise ring communication...
+Rank [3/8]: Starting iteration 1
+Rank [4/8]: Start tile-wise ring communication...
+Rank [4/8]: Starting iteration 1
+Rank [6/8]: Start tile-wise ring communication...
+Rank [6/8]: Starting iteration 1
+Rank [5/8]: Start tile-wise ring communication...
+Rank [5/8]: Starting iteration 1
+Rank [7/8]: Start tile-wise ring communication...
+Rank [7/8]: Starting iteration 1
+Rank [7/8]: Starting iteration 2
+Rank [6/8]: Starting iteration 2
+Rank [5/8]: Starting iteration 2
+Rank [4/8]: Starting iteration 2
+Rank [3/8]: Starting iteration 2
+Rank [2/8]: Starting iteration 2
+Rank [0/8]: Starting iteration 2
+Rank [1/8]: Starting iteration 2
+Rank [6/8]: Starting iteration 3
+Rank [7/8]: Starting iteration 3
+Rank [4/8]: Starting iteration 3
+Rank [5/8]: Starting iteration 3
+Rank [0/8]: Starting iteration 3
+Rank [2/8]: Starting iteration 3
+Rank [3/8]: Starting iteration 3
+Rank [1/8]: Starting iteration 3
+Rank [2/8]: Starting iteration 4
+Rank [7/8]: Starting iteration 4
+Rank [5/8]: Starting iteration 4
+Rank [6/8]: Starting iteration 4
+Rank [4/8]: Starting iteration 4
+Rank [1/8]: Starting iteration 4
+Rank [0/8]: Starting iteration 4
+Rank [3/8]: Starting iteration 4
+Rank [6/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [7/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [6250, 50000].
+Process-averaged run time: 1.6576968906447291 s
+Rank [4/8]: Local distance matrix has shape [6250, 50000].
+Rank [5/8]: Local distance matrix has shape [6250, 50000].
+Rank [3/8]: Local distance matrix has shape [6250, 50000].
+Rank [2/8]: Local distance matrix has shape [6250, 50000].
+Rank [6/8]: Local distance matrix has shape [6250, 50000].
+Rank [7/8]: Local distance matrix has shape [6250, 50000].
+Rank [1/8]: Local distance matrix has shape [6250, 50000].
+Run 5
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/8]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [3/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [4/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [5/8]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/8]: Calculate diagonal blocks...
+Rank [2/8]: Calculate diagonal blocks...
+Rank [5/8]: Calculate diagonal blocks...
+Rank [4/8]: Calculate diagonal blocks...
+Rank [7/8]: Calculate diagonal blocks...
+Rank [6/8]: Calculate diagonal blocks...
+Rank [3/8]: Calculate diagonal blocks...
+Rank [1/8]: Calculate diagonal blocks...
+Rank [0/8]: Start tile-wise ring communication...
+Rank [0/8]: Starting iteration 1
+Rank [4/8]: Start tile-wise ring communication...
+Rank [4/8]: Starting iteration 1
+Rank [2/8]: Start tile-wise ring communication...
+Rank [2/8]: Starting iteration 1
+Rank [1/8]: Start tile-wise ring communication...
+Rank [1/8]: Starting iteration 1
+Rank [6/8]: Start tile-wise ring communication...
+Rank [6/8]: Starting iteration 1
+Rank [3/8]: Start tile-wise ring communication...
+Rank [3/8]: Starting iteration 1
+Rank [5/8]: Start tile-wise ring communication...
+Rank [5/8]: Starting iteration 1
+Rank [7/8]: Start tile-wise ring communication...
+Rank [7/8]: Starting iteration 1
+Rank [7/8]: Starting iteration 2
+Rank [6/8]: Starting iteration 2
+Rank [5/8]: Starting iteration 2
+Rank [4/8]: Starting iteration 2
+Rank [3/8]: Starting iteration 2
+Rank [2/8]: Starting iteration 2
+Rank [0/8]: Starting iteration 2
+Rank [1/8]: Starting iteration 2
+Rank [6/8]: Starting iteration 3
+Rank [7/8]: Starting iteration 3
+Rank [4/8]: Starting iteration 3
+Rank [5/8]: Starting iteration 3
+Rank [2/8]: Starting iteration 3
+Rank [0/8]: Starting iteration 3
+Rank [3/8]: Starting iteration 3
+Rank [1/8]: Starting iteration 3
+Rank [2/8]: Starting iteration 4
+Rank [7/8]: Starting iteration 4
+Rank [5/8]: Starting iteration 4
+Rank [4/8]: Starting iteration 4
+Rank [6/8]: Starting iteration 4
+Rank [1/8]: Starting iteration 4
+Rank [3/8]: Starting iteration 4
+Rank [0/8]: Starting iteration 4
+Rank [6/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [7/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [6250, 50000].
+Process-averaged run time: 1.6466248530196026 s
+Rank [6/8]: Local distance matrix has shape [6250, 50000].
+Rank [3/8]: Local distance matrix has shape [6250, 50000].
+Rank [1/8]: Local distance matrix has shape [6250, 50000].
+Rank [4/8]: Local distance matrix has shape [6250, 50000].
+Rank [2/8]: Local distance matrix has shape [6250, 50000].
+Rank [7/8]: Local distance matrix has shape [6250, 50000].
+Rank [5/8]: Local distance matrix has shape [6250, 50000].
+Run 6
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [4/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [0/8]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [6/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [3/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [5/8]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/8]: Calculate diagonal blocks...
+Rank [4/8]: Calculate diagonal blocks...
+Rank [2/8]: Calculate diagonal blocks...
+Rank [1/8]: Calculate diagonal blocks...
+Rank [6/8]: Calculate diagonal blocks...
+Rank [7/8]: Calculate diagonal blocks...
+Rank [3/8]: Calculate diagonal blocks...
+Rank [5/8]: Calculate diagonal blocks...
+Rank [0/8]: Start tile-wise ring communication...
+Rank [0/8]: Starting iteration 1
+Rank [1/8]: Start tile-wise ring communication...
+Rank [1/8]: Starting iteration 1
+Rank [2/8]: Start tile-wise ring communication...
+Rank [2/8]: Starting iteration 1
+Rank [4/8]: Start tile-wise ring communication...
+Rank [4/8]: Starting iteration 1
+Rank [3/8]: Start tile-wise ring communication...
+Rank [3/8]: Starting iteration 1
+Rank [6/8]: Start tile-wise ring communication...
+Rank [6/8]: Starting iteration 1
+Rank [7/8]: Start tile-wise ring communication...
+Rank [7/8]: Starting iteration 1
+Rank [5/8]: Start tile-wise ring communication...
+Rank [5/8]: Starting iteration 1
+Rank [7/8]: Starting iteration 2
+Rank [6/8]: Starting iteration 2
+Rank [5/8]: Starting iteration 2
+Rank [4/8]: Starting iteration 2
+Rank [3/8]: Starting iteration 2
+Rank [2/8]: Starting iteration 2
+Rank [0/8]: Starting iteration 2
+Rank [1/8]: Starting iteration 2
+Rank [6/8]: Starting iteration 3
+Rank [7/8]: Starting iteration 3
+Rank [4/8]: Starting iteration 3
+Rank [5/8]: Starting iteration 3
+Rank [0/8]: Starting iteration 3
+Rank [2/8]: Starting iteration 3
+Rank [3/8]: Starting iteration 3
+Rank [1/8]: Starting iteration 3
+Rank [2/8]: Starting iteration 4
+Rank [7/8]: Starting iteration 4
+Rank [5/8]: Starting iteration 4
+Rank [6/8]: Starting iteration 4
+Rank [4/8]: Starting iteration 4
+Rank [1/8]: Starting iteration 4
+Rank [0/8]: Starting iteration 4
+Rank [3/8]: Starting iteration 4
+Rank [6/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [7/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [6250, 50000].
+Process-averaged run time: 1.6567660353612155 s
+Rank [3/8]: Local distance matrix has shape [6250, 50000].
+Rank [6/8]: Local distance matrix has shape [6250, 50000].
+Rank [1/8]: Local distance matrix has shape [6250, 50000].
+Rank [7/8]: Local distance matrix has shape [6250, 50000].
+Rank [5/8]: Local distance matrix has shape [6250, 50000].
+Rank [4/8]: Local distance matrix has shape [6250, 50000].
+Rank [2/8]: Local distance matrix has shape [6250, 50000].
+Run 7
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [4/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [3/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [5/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [0/8]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [6/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/8]: Calculate diagonal blocks...
+Rank [1/8]: Calculate diagonal blocks...
+Rank [2/8]: Calculate diagonal blocks...
+Rank [3/8]: Calculate diagonal blocks...
+Rank [5/8]: Calculate diagonal blocks...
+Rank [6/8]: Calculate diagonal blocks...
+Rank [7/8]: Calculate diagonal blocks...
+Rank [4/8]: Calculate diagonal blocks...
+Rank [0/8]: Start tile-wise ring communication...
+Rank [0/8]: Starting iteration 1
+Rank [3/8]: Start tile-wise ring communication...
+Rank [3/8]: Starting iteration 1
+Rank [2/8]: Start tile-wise ring communication...
+Rank [2/8]: Starting iteration 1
+Rank [6/8]: Start tile-wise ring communication...
+Rank [6/8]: Starting iteration 1
+Rank [4/8]: Start tile-wise ring communication...
+Rank [4/8]: Starting iteration 1
+Rank [1/8]: Start tile-wise ring communication...
+Rank [1/8]: Starting iteration 1
+Rank [7/8]: Start tile-wise ring communication...
+Rank [7/8]: Starting iteration 1
+Rank [5/8]: Start tile-wise ring communication...
+Rank [5/8]: Starting iteration 1
+Rank [7/8]: Starting iteration 2
+Rank [6/8]: Starting iteration 2
+Rank [5/8]: Starting iteration 2
+Rank [4/8]: Starting iteration 2
+Rank [3/8]: Starting iteration 2
+Rank [2/8]: Starting iteration 2
+Rank [0/8]: Starting iteration 2
+Rank [1/8]: Starting iteration 2
+Rank [6/8]: Starting iteration 3
+Rank [7/8]: Starting iteration 3
+Rank [4/8]: Starting iteration 3
+Rank [5/8]: Starting iteration 3
+Rank [2/8]: Starting iteration 3
+Rank [0/8]: Starting iteration 3
+Rank [3/8]: Starting iteration 3
+Rank [1/8]: Starting iteration 3
+Rank [2/8]: Starting iteration 4
+Rank [7/8]: Starting iteration 4
+Rank [5/8]: Starting iteration 4
+Rank [6/8]: Starting iteration 4
+Rank [4/8]: Starting iteration 4
+Rank [1/8]: Starting iteration 4
+Rank [0/8]: Starting iteration 4
+Rank [3/8]: Starting iteration 4
+Rank [6/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [7/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [6250, 50000].
+Process-averaged run time: 1.6599961558822542 s
+Rank [3/8]: Local distance matrix has shape [6250, 50000].
+Rank [1/8]: Local distance matrix has shape [6250, 50000].
+Rank [2/8]: Local distance matrix has shape [6250, 50000].
+Rank [5/8]: Local distance matrix has shape [6250, 50000].
+Rank [4/8]: Local distance matrix has shape [6250, 50000].
+Rank [6/8]: Local distance matrix has shape [6250, 50000].
+Rank [7/8]: Local distance matrix has shape [6250, 50000].
+Run 8
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/8]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [4/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [3/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [5/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/8]: Calculate diagonal blocks...
+Rank [3/8]: Calculate diagonal blocks...
+Rank [1/8]: Calculate diagonal blocks...
+Rank [6/8]: Calculate diagonal blocks...
+Rank [5/8]: Calculate diagonal blocks...
+Rank [2/8]: Calculate diagonal blocks...
+Rank [4/8]: Calculate diagonal blocks...
+Rank [7/8]: Calculate diagonal blocks...
+Rank [0/8]: Start tile-wise ring communication...
+Rank [0/8]: Starting iteration 1
+Rank [2/8]: Start tile-wise ring communication...
+Rank [2/8]: Starting iteration 1
+Rank [1/8]: Start tile-wise ring communication...
+Rank [1/8]: Starting iteration 1
+Rank [4/8]: Start tile-wise ring communication...
+Rank [4/8]: Starting iteration 1
+Rank [3/8]: Start tile-wise ring communication...
+Rank [3/8]: Starting iteration 1
+Rank [6/8]: Start tile-wise ring communication...
+Rank [6/8]: Starting iteration 1
+Rank [5/8]: Start tile-wise ring communication...
+Rank [5/8]: Starting iteration 1
+Rank [7/8]: Start tile-wise ring communication...
+Rank [7/8]: Starting iteration 1
+Rank [7/8]: Starting iteration 2
+Rank [6/8]: Starting iteration 2
+Rank [5/8]: Starting iteration 2
+Rank [4/8]: Starting iteration 2
+Rank [3/8]: Starting iteration 2
+Rank [2/8]: Starting iteration 2
+Rank [0/8]: Starting iteration 2
+Rank [1/8]: Starting iteration 2
+Rank [6/8]: Starting iteration 3
+Rank [7/8]: Starting iteration 3
+Rank [4/8]: Starting iteration 3
+Rank [5/8]: Starting iteration 3
+Rank [0/8]: Starting iteration 3
+Rank [2/8]: Starting iteration 3
+Rank [3/8]: Starting iteration 3
+Rank [1/8]: Starting iteration 3
+Rank [2/8]: Starting iteration 4
+Rank [7/8]: Starting iteration 4
+Rank [5/8]: Starting iteration 4
+Rank [6/8]: Starting iteration 4
+Rank [4/8]: Starting iteration 4
+Rank [1/8]: Starting iteration 4
+Rank [0/8]: Starting iteration 4
+Rank [3/8]: Starting iteration 4
+Rank [6/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [7/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [6250, 50000].
+Process-averaged run time: 1.662819399498403 s
+Rank [3/8]: Local distance matrix has shape [6250, 50000].
+Rank [6/8]: Local distance matrix has shape [6250, 50000].
+Rank [5/8]: Local distance matrix has shape [6250, 50000].
+Rank [2/8]: Local distance matrix has shape [6250, 50000].
+Rank [1/8]: Local distance matrix has shape [6250, 50000].
+Rank [4/8]: Local distance matrix has shape [6250, 50000].
+Rank [7/8]: Local distance matrix has shape [6250, 50000].
+Run 9
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/8]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [3/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [4/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [5/8]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/8]: Calculate diagonal blocks...
+Rank [1/8]: Calculate diagonal blocks...
+Rank [2/8]: Calculate diagonal blocks...
+Rank [5/8]: Calculate diagonal blocks...
+Rank [3/8]: Calculate diagonal blocks...
+Rank [7/8]: Calculate diagonal blocks...
+Rank [4/8]: Calculate diagonal blocks...
+Rank [6/8]: Calculate diagonal blocks...
+Rank [0/8]: Start tile-wise ring communication...
+Rank [0/8]: Starting iteration 1
+Rank [2/8]: Start tile-wise ring communication...
+Rank [2/8]: Starting iteration 1
+Rank [4/8]: Start tile-wise ring communication...
+Rank [4/8]: Starting iteration 1
+Rank [6/8]: Start tile-wise ring communication...
+Rank [6/8]: Starting iteration 1
+Rank [5/8]: Start tile-wise ring communication...
+Rank [5/8]: Starting iteration 1
+Rank [1/8]: Start tile-wise ring communication...
+Rank [1/8]: Starting iteration 1
+Rank [3/8]: Start tile-wise ring communication...
+Rank [3/8]: Starting iteration 1
+Rank [7/8]: Start tile-wise ring communication...
+Rank [7/8]: Starting iteration 1
+Rank [7/8]: Starting iteration 2
+Rank [6/8]: Starting iteration 2
+Rank [5/8]: Starting iteration 2
+Rank [4/8]: Starting iteration 2
+Rank [3/8]: Starting iteration 2
+Rank [2/8]: Starting iteration 2
+Rank [0/8]: Starting iteration 2
+Rank [1/8]: Starting iteration 2
+Rank [6/8]: Starting iteration 3
+Rank [7/8]: Starting iteration 3
+Rank [4/8]: Starting iteration 3
+Rank [5/8]: Starting iteration 3
+Rank [2/8]: Starting iteration 3
+Rank [0/8]: Starting iteration 3
+Rank [3/8]: Starting iteration 3
+Rank [1/8]: Starting iteration 3
+Rank [2/8]: Starting iteration 4
+Rank [7/8]: Starting iteration 4
+Rank [5/8]: Starting iteration 4
+Rank [4/8]: Starting iteration 4
+Rank [6/8]: Starting iteration 4
+Rank [1/8]: Starting iteration 4
+Rank [0/8]: Starting iteration 4
+Rank [3/8]: Starting iteration 4
+Rank [4/8]: [DONE]
+Rank [6/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [7/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [6250, 50000].
+Process-averaged run time: 1.659860301297158 s
+Rank [3/8]: Local distance matrix has shape [6250, 50000].
+Rank [4/8]: Local distance matrix has shape [6250, 50000].
+Rank [5/8]: Local distance matrix has shape [6250, 50000].
+Rank [7/8]: Local distance matrix has shape [6250, 50000].
+Rank [2/8]: Local distance matrix has shape [6250, 50000].
+Rank [6/8]: Local distance matrix has shape [6250, 50000].
+Rank [1/8]: Local distance matrix has shape [6250, 50000].
+Run 10
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [1/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [5/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [3/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [0/8]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [7/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [4/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/8]: Calculate diagonal blocks...
+Rank [1/8]: Calculate diagonal blocks...
+Rank [6/8]: Calculate diagonal blocks...
+Rank [2/8]: Calculate diagonal blocks...
+Rank [3/8]: Calculate diagonal blocks...
+Rank [7/8]: Calculate diagonal blocks...
+Rank [5/8]: Calculate diagonal blocks...
+Rank [4/8]: Calculate diagonal blocks...
+Rank [0/8]: Start tile-wise ring communication...
+Rank [0/8]: Starting iteration 1
+Rank [2/8]: Start tile-wise ring communication...
+Rank [2/8]: Starting iteration 1
+Rank [6/8]: Start tile-wise ring communication...
+Rank [6/8]: Starting iteration 1
+Rank [1/8]: Start tile-wise ring communication...
+Rank [1/8]: Starting iteration 1
+Rank [4/8]: Start tile-wise ring communication...
+Rank [4/8]: Starting iteration 1
+Rank [3/8]: Start tile-wise ring communication...
+Rank [3/8]: Starting iteration 1
+Rank [7/8]: Start tile-wise ring communication...
+Rank [7/8]: Starting iteration 1
+Rank [5/8]: Start tile-wise ring communication...
+Rank [5/8]: Starting iteration 1
+Rank [7/8]: Starting iteration 2
+Rank [6/8]: Starting iteration 2
+Rank [5/8]: Starting iteration 2
+Rank [4/8]: Starting iteration 2
+Rank [3/8]: Starting iteration 2
+Rank [2/8]: Starting iteration 2
+Rank [0/8]: Starting iteration 2
+Rank [1/8]: Starting iteration 2
+Rank [6/8]: Starting iteration 3
+Rank [7/8]: Starting iteration 3
+Rank [4/8]: Starting iteration 3
+Rank [5/8]: Starting iteration 3
+Rank [2/8]: Starting iteration 3
+Rank [0/8]: Starting iteration 3
+Rank [3/8]: Starting iteration 3
+Rank [1/8]: Starting iteration 3
+Rank [2/8]: Starting iteration 4
+Rank [7/8]: Starting iteration 4
+Rank [5/8]: Starting iteration 4
+Rank [6/8]: Starting iteration 4
+Rank [4/8]: Starting iteration 4
+Rank [1/8]: Starting iteration 4
+Rank [0/8]: Starting iteration 4
+Rank [3/8]: Starting iteration 4
+Rank [6/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [7/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [6250, 50000].
+Process-averaged run time: 1.6586514977971092 s
+Rank [3/8]: Local distance matrix has shape [6250, 50000].
+Rank [4/8]: Local distance matrix has shape [6250, 50000].
+Rank [2/8]: Local distance matrix has shape [6250, 50000].
+Rank [1/8]: Local distance matrix has shape [6250, 50000].
+Rank [7/8]: Local distance matrix has shape [6250, 50000].
+Rank [6/8]: Local distance matrix has shape [6250, 50000].
+Rank [5/8]: Local distance matrix has shape [6250, 50000].
+Run 11
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+ [OK]
+Rank [0/8]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [4/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [3/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [5/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/8]: Calculate diagonal blocks...
+Rank [1/8]: Calculate diagonal blocks...
+Rank [5/8]: Calculate diagonal blocks...
+Rank [2/8]: Calculate diagonal blocks...
+Rank [3/8]: Calculate diagonal blocks...
+Rank [6/8]: Calculate diagonal blocks...
+Rank [7/8]: Calculate diagonal blocks...
+Rank [4/8]: Calculate diagonal blocks...
+Rank [0/8]: Start tile-wise ring communication...
+Rank [0/8]: Starting iteration 1
+Rank [2/8]: Start tile-wise ring communication...
+Rank [2/8]: Starting iteration 1
+Rank [4/8]: Start tile-wise ring communication...
+Rank [4/8]: Starting iteration 1
+Rank [6/8]: Start tile-wise ring communication...
+Rank [6/8]: Starting iteration 1
+Rank [1/8]: Start tile-wise ring communication...
+Rank [1/8]: Starting iteration 1
+Rank [3/8]: Start tile-wise ring communication...
+Rank [3/8]: Starting iteration 1
+Rank [5/8]: Start tile-wise ring communication...
+Rank [5/8]: Starting iteration 1
+Rank [7/8]: Start tile-wise ring communication...
+Rank [7/8]: Starting iteration 1
+Rank [7/8]: Starting iteration 2
+Rank [6/8]: Starting iteration 2
+Rank [5/8]: Starting iteration 2
+Rank [4/8]: Starting iteration 2
+Rank [3/8]: Starting iteration 2
+Rank [2/8]: Starting iteration 2
+Rank [0/8]: Starting iteration 2
+Rank [1/8]: Starting iteration 2
+Rank [6/8]: Starting iteration 3
+Rank [7/8]: Starting iteration 3
+Rank [4/8]: Starting iteration 3
+Rank [5/8]: Starting iteration 3
+Rank [0/8]: Starting iteration 3
+Rank [2/8]: Starting iteration 3
+Rank [3/8]: Starting iteration 3
+Rank [1/8]: Starting iteration 3
+Rank [2/8]: Starting iteration 4
+Rank [7/8]: Starting iteration 4
+Rank [5/8]: Starting iteration 4
+Rank [6/8]: Starting iteration 4
+Rank [4/8]: Starting iteration 4
+Rank [1/8]: Starting iteration 4
+Rank [0/8]: Starting iteration 4
+Rank [3/8]: Starting iteration 4
+Rank [6/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [7/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [6250, 50000].
+Process-averaged run time: 1.6612193660112098 s
+Rank [3/8]: Local distance matrix has shape [6250, 50000].
+Rank [5/8]: Local distance matrix has shape [6250, 50000].
+Rank [2/8]: Local distance matrix has shape [6250, 50000].
+Rank [7/8]: Local distance matrix has shape [6250, 50000].
+Rank [6/8]: Local distance matrix has shape [6250, 50000].
+Rank [4/8]: Local distance matrix has shape [6250, 50000].
+Rank [1/8]: Local distance matrix has shape [6250, 50000].
+
+============================= JOB FEEDBACK =============================
+
+NodeName=uc2n[186,188-190,208,215-217]
+Job ID: 24894112
+Cluster: uc2
+User/Group: ku4408/scc
+State: COMPLETED (exit code 0)
+Nodes: 8
+Cores per node: 80
+CPU Utilized: 00:05:39
+CPU Efficiency: 0.49% of 19:22:40 core-walltime
+Job Wall-clock time: 00:01:49
+Memory Utilized: 1.93 GB
+Memory Efficiency: 0.28% of 703.12 GB
diff --git a/2_psrs_cdist/solutions/A1/strong_scaling_symmetric/strong_scaling_symmetric.pdf b/2_psrs_cdist/solutions/A1/strong_scaling_symmetric/strong_scaling_symmetric.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..555862692ba26f897cabc93bb5d9659b622f8ff5
GIT binary patch
literal 17608
zcmdUX2|Sg-*ME}j+Oj1TU1SY+zt^69$&x*5*>~4=6IvxjmXIYSB3mdz$`Z0mwi1a_
zNr+S;TI4^^MG3#R-v8(S|9sx})-=yE>p5r6%$zyjc@of5Q4>c=U||CJ1Ms|Z7z&Pr
zd)m6fWM$z9qW~`lI6~QmWaIAX1V`xEI63&j(V&0<TwWgLK(vPnO0KA&=1C;MG2{XW
z(|xwC4t69sVewVfpQLU`vLQLZv5N_MHYAdR4-t-oUSSABXB&GLq7xjyl<VVZXXrqJ
zTYzp=)Bq_C{v<d;(*uy8v=l2Z#hP%d7384SG62yHQSajbs9Pi-Vc_8FdC12O@B`6r
z07vLJ*t^&$dHRDMk>HOc9t+3dk#H**LK!dxYWl*Z7E3A;iJssM1r=K<2q=45a@r0=
zCz3N9wbYxMi#y;Ij!<(40#I?V^R#z>NcJW9IM{f={F58ZTZtBg&Cdr5UZTw|zW&&j
z`MgAtGUFs0rPWrmlG0eg!q0y5-{syMi;lbL{M>vmpD;}5T3wQFOF&<q)A)(OgEtP!
z?Xjz_oHPkIG;@Rh3&n@S+J(&igBaV6Pse-d@`*B@IC;-U*+riY+RpvR9)IDNXP9%`
z?$nX75|;b)SFWabhDzTkyRA*5yCuh*mAQvu{Ibgpc(nJGk|yJ4WAwXu<9sdYjw;eJ
z{U~tzBJ{o4BpYqc-$(ENF^J`G@SfXE33r8fGcQlH#Cms39y;7!6%o2oS5-xls{Ren
zQP)?_Y>`y$T+R1BRK4`Pue(VvV#pvw;aS<;o!-idQgt_p6~*DineC`Ymc`+B#^@|9
z4<}C0$vJpFZ3v(24WE?lJG5Cs+Upg?n;jt&$_Zkx(nE?b(<3D5#_#puCt5>fw{d1{
z>Q%{@QWRIwdg{#o{%vr^s{{0i_&eUM_h`G+`Bh$<2XbyoYKc@TIUS9EL{!?P`uQH)
zqf2*(51gL6#*!Y=NY&J4EEVU_#j5m7qk>MKVquuHQ?9$}Rb4_v+(?@!F<4*na0ENE
zt|MHEa=PjH`@!*oiN_XSK0TiK*8j<0SD{N1aSE^6*jywR@Io!!`9MhVv56lVd7pNS
zwTq5<^Pez?m<$RPOKg-ee56mC_)*_ZK*=?OnqTD<O-QsjrS0rK*puMxJ-!9`w>SjM
z?uN#~^0G6OT=h>qwr}3cPid=gf%TC6URxtoQmBc;Uf3(vy{`ExnvvaCRbETJ3Z&Z|
zOgAMoTyUT9Z1a{~U(dygS*mD;JTWxjy)As-yC|yi!u>qI8oYXzie_shBX3-TKd+^d
zW@NnZ^_XpO!tUaS3VEs|2Q%#XLsXM5gwk%(I<^r5vu3t0+<$myb#=8>F^7!G7jzio
z{YdqeGyMhE>-fIN!h~vz_FulC-shsysy`f3{?>%LJ4T54-MLFFT_H4f>^?tr%%rJy
z9{Un1O>m%~@Zj8lj^p3BrPr_ML3_cm@9Gwl?zxQTQAK+K_^ievqhHhuwU{8Nj(l-3
zNQ^z=rhK%_)Xo2*&=qDsHO}GlhN9C5Sw|r$pTZvLZRl$|H^=3F9-}&zS5_AkZO)~R
zd3TkH>*3w$sfkT-l>O#H`i@(^jSO>yGRy5;_;#I$y<_|t_GO$n>zCIbDcbhM-(a*F
z6^`_KeJ|+{;rO2Z7@6s9ycWYlGfAZr`wjQVkD71?lw&xWi<(PNXFC;NX=S-*(|4sm
zD-SEFDD8RS`+nNa;&Yw;sf4pMu@3vKLKE~Jskt)j4!!$m`YF@=$vF6gsYh#j`DXF-
zjIi#j^^y{s2|Rt*c1E#==k3jCeE~AIIpbpd3~g~}>2tN}eq(2^Z8DarVt+6RvM#yd
zVzylnk5f=QO*u0r=xi_MT_`_)Qvb-wDZS0QnM@TO+-5~i-6@KXnt$ZRwl#XN-NjrN
z@lM$N;dJeAhM!!(LpV-}rED%gT1(BK$ev3?;E~4OkSBa0v@>i`J9Bx5``?eDhPvN`
zcUg)WRY*UMRizIQxg}^TS|W&{wWRQnXVNMxIkDg^=BOlgM?k;%p?#N<<KFzRB-o9X
z2!gC1Z!I5X$@f`1>M54Je96+|r`w&fHjCS3)ox3=-sO0a<yIT1PK@gL89p7aY_9f(
z^VW^&MaJyz@|X~3UW`Cby>>{CEp<o?e^E~QgmT!Vy2?SW=(oW$iVluNzY7S~CwoM4
zD!TSMzExLwY1@oM8@6#LAPsFIA9g!4I>xbDx}s~Jo@M>w=#AAmSz>ZadbFvqM`j<h
zk$S<Ng5KW4sXG-(DL*aI{^x`u<X%_@j}wnb>)}(q^(H!<a?2@pk6%uw6TuwS58ba{
zneXP5@@hd^&L#iJ`xHu#^S62YZY67QNCdV-mM9VZH42%>u?k|c5lL23;*~NlEO%GL
zQd?Y;c@!E))Avr`eX`pX+DY3jEIpUGo(JAk)hK-w5JI{8M}(i;lh~+W=7A*Z=gD6h
z^|Z$4d=0mIwJN3EThMT`lP-~>*!Xq0X4KElnqA1G?W;i=lIeEoW|M5!9WMIk6lYro
ze)bwlGoF3i8+Mmq(s=y7bE`vjvu!fr{btXr6ZE$f-xJ0f4TUMo%yX%q<lSU<OJc}Z
zk=!Nv(`MMjpE@>PP~g6Csq78U!YqH!8;625j-Z6K@0#tRiLjC53NUOIJiK?b<+I1z
zo3B+q<R3wnw{f!<oRzg6`rwn%bhR#vffVLpR(^jz_+B#Je}6|RwW0Agiq_ox&4)}E
z<^$(@b-20>6)q=r?b-ZL@VsqicH_<yA>uZ#3`Jy}O>3)^)dXbh7}Ab$&8nUr=#HrE
z>=bKD!De21oR6GI&Pwi@F23mFczBPUlk^3{8B{m7?^*t=mY-S``<liKsc}<!fj%?y
zH|0e}o}ciN*y#>g8B5#blD$FRQ=xqpC5c8Z8zJP)7)rDNJN9zHbv9%I9&EyY*^K2P
z@3Wj#pru?eiecTXQh9tw6HV_zuQK(X7HVpr++dV_*azywAh&Dr7X(E(srp_;r@HoV
zswT^bb2@D$WS-#YN)6^IKiK?<b73v<Xvk<-Wh9_6sDB_zPt&Ik$x!i~<LJf)<G}MW
zS9T069Hd7k?QpJnO<icAI=of>3a!jku|x-dn!t&~R5>w*Tb6vAONXn{*-r@S=iJz)
z{iNlj)H}VHzQ-5tUA!MRAd-BkDdb4#tb@f_xw^ZbzUKPh?5#96WJ!AMfo0ai=6*kF
zN@8l=AW>%-|IEmQ$KY1xRndLb-UHE8t}r<kg<y<a^RALWWlw$2y*IZomut@XG|>-K
zh)Yt6VyK>UoAuawJ7&A^*YO+r9Y?yhJPT)d%urmnQ4ZfOPA^y06?s{j{Ui0HXJz&N
zS0Tq2DBjqayj(|KE3LUDqY8t?{9Rr+0~w+KieY?&y;5PrxKrOqz<t~BiTQp3oQ8l>
zsm3|K@3l%YRbqw3U1M{l*^{aTktc$4ckJ26p^sp9dbepx@}0E&K1W-G^3A$(mMeyP
z_RVp>IjpSRenGm8O~~oceG${!gWI%(luDnLHDCUCrF-v{X+=M{32$FyMtH2Qw7x`{
zrHXF62W>!g$kX!TF@ahqojo%qoCVP?G(~zErA1E>p6<1m{L=$P@1VEuaym#uq(Kus
zB#{PaUh~L0!Mx{LsF_I3{;I;)NIEf(^ry*u9ZyOgzqztE?YO>y9s!NT#5|7kmMFW_
zg2+SMVl>#Hsx~F0JavvP?1~jz_+ek(#szumRP@$?j-hbgKJH2DT|?Qoo6{=GexAE%
z;hAAX<xh%tiYhy{%TYaXhq-R*UVFKop-SO<ot8L4wUu+0@Qa6)&Ud)peL(z>J(0R+
zfkN8zM9Mnj0~xrh^%;x%`}nBpOA$AL@tKDmv)(XXGM_@XMFhX#sq8AGp;{X&Y;eFF
z?Qqueoq?9alQ;x-l>z&{J^bC5RJHYY(_x*@4DjQhnuK8*ChiaCpDfEPtlJ&fc>dPL
z_)dN!#>208Uds*ijyR-D79LCC%6lxDkZl~6D%Vz+#7(MAI#0;$6gM9GiRh54IwI=I
zd)ecSboSvauJWLZHAg*KPB*#4dhHvmKk~-1tw=9u@Y3BD=^P^k&Y%8j9j+w}o9xT)
z6e`C=VvN~?X)X0X2NI9o)|wQ!c7AvN(*WBft?vhpg}%^656gW#&-`)w;QY^yi(eyu
z6dXKgWL3G2^q`~CYUyDl|1Le;c`YI(ia~kS7iI-NIa56GSzC|P_2R^KgZ2$cq0s_U
z*GUheH3F`;k}h0Qei5ju8)h~&c&@E_JV5?jyyw0fIzeZJ!(wbD4j7gWBwm}@>m3<|
zDlz-e+US}Y#=M6cE%a^9@Ok~C+ot&p9<=JL2kMHluZ?!hkha!IP&98a*~K4wZc|c^
z%9ttwo)wf(f7xv0<@aw?1+~w;D|=lzC{-FQyBQ{Fx*xs3eD9gcN`C!v=yb;@_h1`y
zVvQAv@gw>yM{=EonZrKQw}+c#`x_?i7NsPcXuRE|dDr*psFHbk?9Z~Zcjk|7Fgs>>
zZym8gC&<-e!~M%5`b+7KTT_sg?i715P&${`8yXsD)vhwU;J`;#<!35y>JuZ<%Qt>F
z>++H(cglvMk7<UEF4$4&p%<gvMwSqf-52hbeA*^|7&H4$Y%c4JyPpzc`q{j&iMJ#T
zlkBYyA6PV#LaqkCY1OO{aM@l!DU<i!!k*u%-jGH2QA8&(;ViB{d4INI;%%mgd1J-y
z5?|FPA+Jt<df3_C;C!?2fNcE5Vxd!PRWlt$n4)uS1|N66!5u$C<5oE`k3931p)=}!
zz{NX<Qtgv(8C#`%=<;V|bIF>1JA5v|YOk49PVA3lohQmAFMN;TUkv=5r}w;UA-awX
zp=0+d8KN-%usiCUS7fE+rQP(Q@|4hqQ+Ge~8M!jQUFeOtggfG0z9VxR;(l%5ORvN2
zqCX7EKVhy1dS$;3xX_ou7sOslX%@Gyh+4@_r)+OK!$2hUX}T+w&A~egzPm5%(ugnj
z#d!@B{yaaaD1O}NRes$qTu)B&C1j_cSM4)vS-;)oQv>~e<)iadQT&)k>o5d44puQl
z!2W$IVelvuq+TJCf>kXm0ru9B8YXhaQlaX4SecRprv^j)1BzmXlUP$1CT&%w{nVj!
zkErz-Y!%rrb6ryIzR$I}jNvAB!>+h3jq6YZc|KNAgqHjVHB~jxjHN`;g1uh$zT=3T
z94@}@;X(n6U$(D?7L^)p^Q8w^5vo)l-p`tQ+)Tv{e826iI$a`I#@>hap*h!ytE_R?
zZx#u^xV<2z_Ob(C`sOodbkwrCHM_Db#v&q%o?O!Twms$(JX<bb7#*hNsojlb8|9Hj
z*voOvowtuw(Xmc>56{=&d?Cu0%-%|{<*D>|yTw24h^?Zhy}#m(X3pplwML$33b#9=
zUtW4Y*Y3V*+OIe(%S7U#Id#KtkM}+9(1Doi{S?`QOiz7ftbaUH9P;kWm&ra_gH@3J
z>G1CID;{%-WBkE;JzvyU6e9*$D{7yA_j%wqXfQuuK+jSqXQuzy_hQ;M32~&rm-(+9
zmP5GXgJs9OW!I7LO5fA6%S8(L_d!>U)`BFgJS!g(by9K8H$^wo5mmN;M}HNc$W|ia
zkQH@>Ic}R`>b%C^H;2|-i+fd#>P>99^YP&wm@hJ0KK)`S$<k*dit&wF{$`Gf)-bwD
zqE!mFYvf5i-r97>UznAl)IYhhd^aBRk$x4(;@12r?Z-Ryi3)#LpA6mp@SVG|>#r}c
zt6z_QDNh$gQr9y|{q{NTkxEHEW6VAO+a<m2QJ2}{WBdenSg}Rr+;}7?zx&>^gB5!y
zTo}}x*Ac@yhAtZQFGiAH86((6#^pk`&|1GM_DrEah3%`36c%a-lD(X+F>(Oipu>aI
zJRFuhRbQA+<W04~TpANK3RMfW5}fUnzMa2aygRu3Th>CBUXa`o>a<D`&K!Cx3fkGt
zA8afx2OKlH`-(6W<oBuD&tpQ)en6Y*g4bL7YxIvF#g8o<oBT9M%>T+otT2m{j>yh@
zb|AN2bUuK!HP}gr%kN6u#?BPY#t0^f={I&h^tYs&+|R_bE4*iT^Mrb4ZhH4zd{ObI
zcN>gSHFh{@JVQ-aQ}4c4#dpTYdaZOY>zUlxf0>5SGLVU)d?fW0r)kZ%XYw7J!rA-2
z-(kVAW$;mGz9~-JC0II?>C2l4_j#>#rt!Y?T*=t0jyVfAPTjh(>d)?X7>_f#_h@rC
zW|n#}wnoy1?0e5M+ZPy_V_M#P=_0RFZQjDRvG}|$_bq+7T2dlHM}oKWWqYEzOn7ar
z-{!@Mce@oPM(rqYMw)CkVr;wjvGT{Z!8((g@VA)!xv{(No@7?MIINLoCULJLU-EkS
z(;nUMQoOy^pvIh`@FnZq=<L8Zg6tmZjzs_YmSaOl{LZEJUBB^Mxyg|jeVRbnZm7}P
zops_x-JN?UD2T@o%&a51b&PKm?jOt(RXy?^QSlv?ax!-GTgrP<;M1QA&uU5^2MeCo
zR=8C+z%*LxH$*xzoq+vf^Wh9t@Z%bfQMBTrvsW1T<Bd7KC7j?)5uY<Ay)3-taw|*P
z?hw7$_Pz9}A;J7Iw#5B;Ixyiu<8P@!4N*58uCZS9RoeQn<$_x7_^jc{134b?`Y>fC
zfiR(5%*c)W`k|lHZ%eBW-hGxQx5v{%QfwVI*Rf$S*nijxaj}|26%?(?tUMK`bBi`l
z`{!#WTdsWf{0x5pbNA9Wmh^lOhYh%{)fi)GSVhXn&Di&K?BR!mj?nk4uB=r8V!H&L
z2mxmb_*o1GE^K{`k(RIfGV##1>qtWlLvOCI+*{iV=h~x=-xH1Z2uw0=ZWKE>-*|nO
z=TxY2j|ApO)OInO4O)c*FYD}uI<DJh2eRIrDYCyb8TIV$l{7&Q%>!XW8ZWD`K^6P^
z?e@#m@_+t79mk1bFD=NpD*IA0dxu$Om`r-@R}3-KHhSX?$A}l{&jeBCEFNhp8XNbs
zn7zAFmOA;-veL}hhRaptxI*pIsL*Q$eNkCA&PFv1;CuRtZPfeheED;BmD~zHC%B6v
z@4-h~7aO0xx^jVQ--@gDdX&{(cZ&@bw-7Kr({rcGHchkfkz3eIHp2{un{ghdY3yO!
zqo7s;i=F%w^wfg7KTp&;9X&sb=*c_2ZJi-n$56)o!zlxU#~33wC|POoQ`y1$2bv9d
z8=_mAyLUW%IDCrktryoe)06STUR(+;7jt*qgm!XR4wK0`-LGSCqW)nXVDPu}kSFWd
zl|joqT$IM^w7jkvhW>{g?7SkAGB41*<Wu~#wq2i}Vy&Kj_mpW~7(6Z65ur={xjiKF
zbz#rb@F@!oxleblCw)#YWKI$l9#BR<&_UR}Z*Dlk9WxkUAP(oY#Fcn^T)*%zwkPWf
zTEaN&_|4{8qeg93OG{HfYt06ltyjfP$MZBld&PZjrl)u`9dYcZ862&>4m0bRU})q&
z*kC&Pio{L43>BYYC#(fnH}ZbybA6)xeZkGcQSDuepkO1NOH|Z=e_zd1Bv%;1EnjkX
zHJ|7_Hucka)vt94bt4LMvXXvH72NpAz15wq#D=kJ2M8kcCgZ}}6X<o-FO0uVx|8$f
zG}UeyYh~SuPZ{XISgtdNFM3p!F&Sv?5i7sdl(g^WgDs>Y9aYUyALOv0U)x;%YuhWr
z@9>+!j>hNLIe7G}fo@uOpmj@=lxg4a2;75!!N0`I@nHg?!CZf6w{nRsAE(0a4oP
zoRCo~t-wV<jf|2Zlx%z*7N1m2G?f+i?N)JcwJ|wlXhZZBSMqeXhjhTlm!#}$;{!*+
z5ZX3N&uFsZ158TA!Pm~m#f#+W14k{PW`>7sNo33p>Iefv=-PMy<m*4S4;oQgguKL2
zNCFZrjs*>&q=1tQg(Bb}><c1L$A;wNLdL};kVt4t0DpcxSpj?yP^bssIsl20u{0P$
zl?V_n2+aEpA}%Mv5GpQ?jt&5dgRnOXI0optua}J-fCUNQ$=$}+8BPFGOBXH)atz=I
zL%0-pwFPh+u<L+O06&i41x$Vy5J|xi9&iK^hVX<Vya3fe*?d4Va0CDg4*>=M$cEVO
zVo!1g#F2r<l}O=Ab^SduE5u4>|1UChw{h|XTr7&}zmz5%q3CM|L6A~tKsMA3^dgRi
zPF>12UK$QAPR=A~s7;q20l*2MR~I`)qLaG=5Ie$<<ltcfJ?Pl@li7l!uvp|uo|e}W
z3pgGK?EgW4{ol5M!AiohI6NGOM!_Y4Qv)xBhk-#v!zBr5I0lIanFMmcp)uqF7)c}?
z3q}cwF?bwQ;Xeh~$~-J!6beu$%K-=40G%P>1aJl<;L&gb1`9HQL?kifAW5c*0K-7U
zfItBHj>iz-Qc^fjjsTa$fw~w#7!-r4K>(gxJWxqV;C_bMM4^xv@PGmtpaV1x25|#X
z4aE=@IM6v1@StG=m{vF`P#(m1pxs!^Vj2N726;1~fCqI!3`B$lJ%9|nB!OH3s)PlV
zaey)`s7%hop@9}*;dtO`!-HoWpp4uYXatG@0gK}Xi4YJ$7A@!$4~Qd+WtloCCaVy{
z9~RU^q3{H96~NQ7`oPE|0wfAmCkKoanYUj}Ej~a5LQNp00A&#S5LJN6WlexIfIM(O
z<DnQ5*D@iCEdbCR4=gW8evl^oY9A^^PJ}WRg$pA<t$+X#2j!E~ml^<r1Jz#s2hs+z
zB0$QpDxi>(IM5d8cu6ys<7LfQj-mdSH3QO$KLV5orJ>L;NJEwbSvf!+0VPFN1V|ZH
z2Qc73TfhKAnz16lmNjEJUQ~!h#ekky28g)jd`JP71Ed^4fnZSl8yH!6Y}N$}K3Gw}
zN^1ZoqZ8yc7t9WFnk^i?bcMkl4yGwN%>fR^l>Fof2TB5-U|=mqFKs~{aP-oR4<a1B
zbhpA24puyJ4KFbB7oU9K0QMlKLCfc2D~roNn01Tk<ZS_r4EYV*D_LB?!MYAc9ZLIs
z2L&AG*0@@v{^5v+i>P3%LDH%m$-2ujCwX!<ES>$!PmfS>&H=fxu+Q$+_E*mGmkezw
z%JMSQHqgyE%;1_y;FcK`Xq(5j1WIh1X7--pD7{P>tG_GetV&{QywmW(qw)-%J8ah)
zM_^X3A@N^)mY5=}G4M~7Y+yZ;@D}z~ErINxk`Ik<rDALS!#A~m+lJ~0#&yhBu)3`2
z8iNGe?O&0w*d;f!e2B#6ns0q6pEHEZFTOa@FC;K_Zb#o1Ro8@aZ8JljXzYd4C&oT{
z-MTU*<KE%u_|iN*F>E+)q?5yTGW}@ZLp3Wip@w&-9i{J#>PvCb;B|6S^TaZrOZYSO
z3Z__mVcApJd6lOdCqYLb?Z4(Ag0;*2e&W%Q9}(!6uG&`a26;zz${tWL9v!7o596~-
z<Qy4Nq#N5?>ueqxe^`}LadN8U!4bO2{ui#zMKsqhw#$e(7LJn)B#4}6d3l<ray|}U
zd+MgtF;}=-U|Qqp8H|t6q;uoj?&jCgKVL*{(mH?BnfLR{0%DJZb)})Ss;#}QPrggI
zvF@xM`{S~@dkmXr+*OrDd}E%xJ6P9F@0Q@c#eeTNDxPc41+|z=`ucNpKirV*>zv(b
z-b>oK&akg@bU;H!&);N^)7K={lL5aUH5+;+D}@@ap8c}WYw&=bRzaZju7Ir_|3(CA
zPAYU@E*5u9@VJxD{vTrdLey(-ywqe5N_=&|?^NsdMu$C2n4Oi0ChZxG7Z{pV@p7^(
z=%)gkrI8iXhn_k<3#;ONRm^YDrk3!ymv#D)=%xpTrGsOqSUvK-`F_u*_xS#dP1o<p
zJI!IX(2<1v8iyf{3k{~!a{6UjsAJLZ+|z{8Duq*n-Q`_FRT71rUWVPjFWkvAQTE>H
zwK1==)$?o*yBxK>O^>S?b#!8BzpBtVm0~Bi{5;Wdud4Suvo5a@%Q^yEwqPLJamlCm
zPY5Y)pHwV3M6?Ek$Rmxn_A69pXP4riemzkho1`24k+<bB8+Vd_5<h>TV^hDmJ>2o7
zp>{*{ml9^S+}W2rd~xCQHfLhIdh&;eDN&pYe(@cqG#~JD=gbalep;*Dvww?|L_d|C
zOLDzb3o*iq?N;QW8`*}Y>c}*P)1=b6IJmD-#&k;*`t7J>j8IDWhcEqNep1Jo#_92q
zyc{EQqJgLI!RZqLA2x3DQCzTkKF6b4F)9GxPPe5;-*z&2`1HYhek7jT!mPeZmlP$~
zc-zKAvo>GS%P+om)j;3tx_tI%e(H(ggO=j!$ZMq)yDTp(Ffjk7U~+mgQ7XI)<5D4r
zkGrP$y?E5}#2r5j-lV)FqNjbl6tk<7B}knC6<07MaO3LDpz3g?PsG>b!L{%_WdnG7
zAaf{bA`2hGCe59U*<(Yni>gsYQ74a)vgBw4rM}m~PWx_mm@v^03d`k-dWVS;NfAAg
z>c2(oeKX!dC|2$b{FQ-0SW9MV+=D$H58`0bj$NC?Sj%48v*}2-R%jYTaK)kD@nre&
z9m|((z9v0tuhX8-pVaqwUw_iv4zD{0pV{liH<5TA&Lx+$2TO63&klv*X>=l=jZVz6
zxribRD!9$B96!Vw|9tDnR@OwP6I_G#3twq|o;BxNM}jL)Xv-Kgux0)(K?5yfg92^;
z2)pIgM;|YD1Q-}Uul(-mrfhIsv@#6Uh&=Wnx@=g>iq$)$<<5bd7P47KB`nO70zxao
zBL(CE@;1p^YDJQ`CCRNmMUTEBs*=-|+m$w_ln_D_t@|;JzV?{Z&mqLg`@MyXcVEe&
z6$^L06;Wy=>T_LURb;UliQ&k}&`UL|9n_y3pG8x-dPZc@+wGX9E1^>I(5T<|PGN(}
zM4Zu$=qRUMY}Fjkooshlk8~AL+~hJ+NjNgm6necqC!$iNUVfj>H{k?rag#=(A=kkl
zaS`1|NmQcOu~TtVoyNtT^rm3}tY4&lXv~fDxbvQoCiYB)4e!J>pjq3kwWO@m5{J}Z
z_M07IxyfC2x1AKUGrq@5yJY-$^US`Vq<-_ZLzO<X=?cR-iXSVOZmQj?uSd&Yi4+M;
z7c1O$sZtGd-)1vQ&um>Y<+gdB%FWnpH@*^6zRS05RU*&1`^w$h)T8~OL8_;Yt=9d+
zm7cUB3`Ugq)U)z4uHy=R#Z{lZN>le>M$<jk8Pk;}&VOfC=;&*nqXf(D2*)=C-UAmZ
zvz3#SM;3Y?Foj5MqAQ5KkZ{qIo*8v|r<mMa46iR?e*5wCz6!Iz*p|M9iz9r7`f~<3
z61GZKq1!vsSU$?7xv`qvy7x&+>{^m<70u3&yPGg+6n4in(PcSr<9XS184m4WGS8^J
z>t<W`+Pq6Pa)9_<&Cx;dG~;=DrGc#{L*e5ai?)iz;qh!}vv$h&j4h1j*1gxa>xZA(
z9oIugN;j#`efYBYZI2&8S$yhc&aqG5oJ|s~L%OtPO#Kgr$X~pk<Tn|s6!=O<hVLN{
zECVegzWer(`gg>jZ+lt|KF+ty&T$toxBak8UPprKfC|8_|Ldes1-EBv;Ix|-j1=i$
zuxI-OFMw?;ax?m<^$64CX)5~C0JWV|@q!09OlkJo4d|U$xcACOyk&rj*-PlevrePr
zgvW<nVl<6K1~OO!7MPp_qjLhf?5@nx+kNL>2LS>$_Zrg%awPn1@x!3+=mDz=OTl_(
zi{LWqm$2DHf>T?J=HbS!1TPq0%Kdoi4P{P@LTiNs47WAig5$$qyH)*VZ(46UF8D=W
zKd$ygHREOh;$5NVa+ch5Ng_l{oicR_gV-4>@1W@fB)z4&UeuL|Yc@}+-ffDU$uju-
zfjKT^BLDT7af*#f+Z@*+Z5=xk0}T7W(uM=885N4QzeD7%LgvSP?VodV!>hn*#&(ZG
z8)-cxf-rleRn;xfpIP_DV&|;DEu|Od-(PxY&@j8Dn7u>1K)r+IjyP<SKAW6r5`u%T
zyJLD(I>&2oYS4b#JM7ylgEcn<yUx+@WMbfYBMOZbwrB&&Q@ft}x_Hxu`3Q!c$f!&i
z6uOa-HZ5nC)H--UE3=bZM|YH>;X#AuRV&S}SCcO8Rf$a;KXyxL`b<r%th_p^<>UlD
z%p;vjaD*pQ;9)(xbYauRO2<o2j7P0&dCrPN?58iz;c=g`V2jV-zA?nrhkVhBG_*~7
zRY=wS*4E+F<nR!#pf>cq{spT#ugQeA!?az_Keumr@Y0SW_N03DEmlSs$(qoblmn6t
z2R^e&G1Y|HUAZL2VCO8D9T*i>)0tA7Y17!mV?Ohl*=X$O*>whD9jhA!&NhD?2-Wji
zOq<BA-SnuMrkx|+=&=yrh4&|~5ZUbHLX3jSPNm67xKUo=*vZRP3S7GhQL1M0Mu7$<
z%q=g^<~O$T4qO%ccuG_B46k1J#ceU-4#Ybr`|*rLMe&B2QH9~xkmtXYxO4CvOt;`W
z6uT$5UKBAfRL<R;*;DZ%J-q5f1_g?49d6dKV==(>^jB_h2EQ))z)uqUKlH&hVt!OJ
zUhiY4^d=d^$(S4YwVn+zx;;VqS`z<^mGI`ULr6VcOCIZcgEJaAl_tDTJ|v+F?%<p@
zH5+8szR7sw-EwMAU1m0Y)7R2l1;fqU=Xp#y_|sHs&<*vY+Y;I*3mq+vC%Dv{_dlJy
zyGOjql*(ZHbIJGTX43-1i>O2IB_*7Lahrr~Ij;EWs(R$HFI$aF;a%@{XnIA2>BXG%
z3wox(Aaz8{Pk-k$Mi|95`fR)EfMSq9>_w@Da?<qRT<5K;Kc7Yrs-;E8EX>X99?5Ti
zAJ-^`9t$*`CVlQ{3Z}<?NdGbNgn<?75E<J@bK{xRne-%m6aPEP7Z}@6O}w9(QtG(q
z_I{IPuRbrp#vRu@$i$d-;vmnph-CbWjc?QUcyQ1#y)LdhP!`0om(wkFmg2hhXwcX~
z!Kd3*y}diR-=AFPrUCE>uUU@J|8lq#*Vpu+RHUuw*gbvPF?-N1arfCb3%#@tXs8*(
z6-ef<uSTT8#r9VY$K*NJ4PsEI2C2>--`(qB`bNPBu4$)-OY{2_IL-6a`y$=L3q6Gl
zj`_?#B18s88(yKfobL2GbZ?%pDv(Q3Y{|E*7hvPmeqzJ##Iz>~Em7htM}Uv0nI_sq
z6HT^ZSfj8Z8oVL7mowToDY8b~RExy>uz$})oF|<zj?6(#_=<ZR?9h>P>zpvnpL!F=
zW92O&#EHn?F+GE)<kWY`pHk4MnI|&O_!eCl^w%0`-|~3qNL$NPt&w%Z<zAu!{>kgX
z%Fv<j5d|Y7Uw=>@w6+LcN5R*zqfx+D^j8JfiPi*247iJxT+5kr8dEbOTb*0({Gs}W
zGR>i&A8+7Zva+4tyCg{VqY+WNFYg$tDTdp<?vh++xhr3fQhc5+Zkp)V)c26pw-$}s
zSgO|2C!1cw++<11YM;2Xss_$yb^fc#XR`~_9d~4&pD!<QJSdl56E;EHP)M==<a~Kk
zkXG&21CQS%iVE1Vot+AL$7g_-N!xteXu>D=T=DbEyS+H3e{?BNzNfm)FFUvn+v^y=
zXyAJLE894IDJDu@hKX5@p9;Go5dQnph1-5sw?%ZB@Z7gNQxeCUG-Yrhhj-XeJ}^ll
zxqP7ap(}PIp+wyG<lWLr#EsaRFvrl~`z{ynQe2XyQ10!@(O2f=>Fmxl6fPXtB!BJ>
zk@>mZ?V!X`-jOi4tuU9EV%v39wk;kVLK(Q~1L50WB))&%Y?#@Yur-L9!bEaM?`ZGG
zc)5LoT_g^4cOI^FD;=9EpX}GDtS4;a=GaGQ*6v1@d{@d!nR%q{h7D}qtsCuA_r?{!
zsrS4!T}-|I+fRyAb)$r}@&TUC-&5;1NB~~}-ze$m>#LZofNvn-{X%ZQA$A3Fvj`Kc
zfEa+^j_lt06S)ES2N}6p)AVYX0r=^}v3Lv|MF6%L3Jtaw0Lc6uv;iMA{Q_;kA`U3{
zZ-F+zVTlNI@Bx^MClTN=q{HBe<m}@BVKd;XD)6t1A9(f!s1X1F5R1SDaP}e;flnQP
z8gMCKqS?Ttz$w*k3Df~@SUAERazC$!-2jLDpAJ8=*p}~B{T`MTf?a{#tdSxR(SLw#
z)|Kb~57-6+{DHsymt_7RcK!#zHUPxK0+<gAK7J#6l`(jj6a<0*A37SiuQA|$6%>|T
z@)%Ga+@S;BTo403Aea1q5n#)CP+2G}y44qv8ys*6lYRbpNyr}$p#;Ep4$v1o@R;Es
zAO#A*m5r9d!9W0D1OQebzy(YJ0?<kV<Q9j387PMGfzKEZz6d0M`cRuvlE4oyxtIpU
zWM(h`1p*8~ge;>GcyN0Ps*3}RFSY^7f_}-*z$1+Yu6iio2vX!&3Tza32txq=|HU#8
z6oMjSUl=K<9u~OxAzT1*^#fj@00Arzez7DpvakV;K^72*fk0tUpEywG4<H005(ViG
z)Dr}Jk+EAcOb4+BHL+NoJSI?e5Ma2aG5gij;sa3l#U_AvpN!c+Q~}DN;e^y+dAOEi
zKrL)pO5_%>00RSTlf?(=!LRn06@;uzpe5-2S1S-vNF<D$x72_XUJAY(AVpZz0Z0%2
z3}jgYy?}ILMIh?~q!Y`5tSd`80qMia0LhyC4%&cpV>v*a<B>QphJaQ`2i6Hv0M#Pv
z1=uC=E5fo)EXR;gz_`Q6G4#A5kjHv4A1d=JKzad<Dir^MYsd@q@>-_|^lEX((1epA
z1pwFB0PJfq%?^%Tf=lcH5V@EJkaaR#0$W;#!SR^9<B->AAfLq-XxRo{L-LC|7@Nf>
zXn7}}bwMxW)qDx4A;ED=JuWW$xMh$A>;z<E6qG*%t^MTo$Xf-NrO?ZYH4nTmbEO7Q
z!zHrRfbK2Q@w@2Cvxg#T<*kBW9~C2%6+wFlI~zC(>I0#O{*T1~F7&$?=06rgN`M^+
z6og`V`4xn(Ft&e_xWan;od~hC%%To7X~`l$=zv+WD6ro}$$86V7dN!!52z8pYy&k<
zA8^~@_ay^3xV8Z%9P}4#`%7*vJQf%UfBD1-qu2Zzt@rWu>qodp7~@WyaWj?A{fQkC
zIT}ofB1NGTrW@|lbW!V7NoXtdvYXX!;M<7X6&&6gM!#YA7LIpott2u#|7)pV=`mRp
z{o<GI2rU<T=#C0l6d=`wuDW<00#`Q38n*P28MqGldyJLA=iJbB9AR)}#>?H4<nCe%
z_rpj?Nub5x&LonTuQUST@heNh)5l2!h5%P*><`(2uftc?^0Ie?+uGQ<fnJtc042%I
zyLb{+ps(nKRix2KG{D=DXrv?tEX%u*Xdxt0NMwbk`ujLI!oXH71p`+hmVRK*#DNvY
z5&kO;m|WyMIB_Wr1(rR)GW7eMhQorZ>#Ngn5E8tq93H=>Jsbp0uP#Rb7T=#~;0+jw
ztJ5$LVhsI$(Sk&yp(X9lG)N4q)6i%LpZ=p93YZaV`H+GZ)j!H13D6#}It>f<t<`DB
zwf#!s0r<SC92&hwF3|9;mLC#LSlb5%Z1=0%!(f4Ju&NIXAbNEg7P&_D;D!mfeX*)8
z77J}B(C@dLv4pkda7Y|jp8qU|1H`OJ!>?0L5*Q(?>H;yZX%CNu5dJ^=05=%crV-Za
zDHgi<{%2hjuy9t_MPYE@__3-S=wfv_Bp`XsSV|$mJ(EA{BBfAk<t&8*muXg)gRVdQ
znTA55z?HkzX;@(6tWJ|!Lmvu-mRwT~g@ra`==W<ZQ9$w6$OT1!ZfN{j4vm8DeymEv
zKzBM;r4b;z<j*v4t7}ca7!0(0{#g!>S~)*RJ~l4y4nE}nam3Ii5Lg`GTxj6w2`#i_
j`$3cF2+RsV>7xDMOS17Hk(V|Uu$?4f0s^Z0)L{Py=ALi|
literal 0
HcmV?d00001
diff --git a/2_psrs_cdist/solutions/A1/submit_strong_scaling_example.sh b/2_psrs_cdist/solutions/A1/submit_strong_scaling_example.sh
new file mode 100644
index 0000000..980dd85
--- /dev/null
+++ b/2_psrs_cdist/solutions/A1/submit_strong_scaling_example.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+#SBATCH --job-name=cdist_strong_4 # job name
+#SBATCH --partition=multiple # queue for resource allocation
+#SBATCH --nodes=4 # number of nodes to be used
+#SBATCH --time=4:00 # wall-clock time limit
+#SBATCH --cpus-per-task=40 # number of CPUs required per MPI task
+#SBATCH --ntasks-per-node=1 # maximum count of tasks per node
+#SBATCH --mail-type=ALL # Notify user by email when certain event types occur.
+
+export PYDIR=./
+export VENVDIR=~/scai-venv/
+
+# Set up modules.
+module purge # Unload all currently loaded modules.
+module load compiler/gnu/13.3 # Load required modules.
+module load mpi/openmpi/4.1
+module load devel/cuda/12.4
+module load lib/hdf5/1.14.4-gnu-13.3-openmpi-4.1
+
+source ${VENVDIR}/bin/activate # Activate your virtual environment.
+
+for i in 1 2 3 4 5 6 7 8 9 10 11
+do
+ echo "Run $i"
+ mpirun python -u ${PYDIR}/cdist_kp.py
+done
diff --git a/2_psrs_cdist/solutions/A1/submit_weak_scaling_example.py b/2_psrs_cdist/solutions/A1/submit_weak_scaling_example.py
new file mode 100644
index 0000000..6ce23bb
--- /dev/null
+++ b/2_psrs_cdist/solutions/A1/submit_weak_scaling_example.py
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+#SBATCH --job-name=cdist_weak_4 # job name
+#SBATCH --partition=multiple # queue for resource allocation
+#SBATCH --nodes=4 # number of nodes to be used
+#SBATCH --time=10:00 # wall-clock time limit
+#SBATCH --cpus-per-task=40 # number of CPUs required per MPI task
+#SBATCH --ntasks-per-node=1 # maximum count of tasks per node
+#SBATCH --mail-type=ALL # Notify user by email when certain event types occur.
+
+export PYDIR=./
+export VENVDIR=~/scai-venv/
+
+# Set up modules.
+module purge # Unload all currently loaded modules.
+module load compiler/gnu/13.3 # Load required modules.
+module load mpi/openmpi/4.1
+module load devel/cuda/12.4
+module load lib/hdf5/1.14.4-gnu-13.3-openmpi-4.1
+
+source ${VENVDIR}/bin/activate # Activate your virtual environment.
+
+for i in 1 2 3 4 5 6 7 8 9 10 11
+do
+ echo "Run $i"
+ mpirun python -u ${PYDIR}/cdist_weak.py
+done
diff --git a/2_psrs_cdist/solutions/A1/weak_scaling/nodes_1.out b/2_psrs_cdist/solutions/A1/weak_scaling/nodes_1.out
new file mode 100644
index 0000000..e4f7414
--- /dev/null
+++ b/2_psrs_cdist/solutions/A1/weak_scaling/nodes_1.out
@@ -0,0 +1,158 @@
+Run 1
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 12500
+n^2/p = 156250000.0
+ [OK]
+Rank [0/1]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [12500, 12500].
+Process-averaged run time: 1.721375777386129 s
+Run 2
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 12500
+n^2/p = 156250000.0
+ [OK]
+Rank [0/1]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [12500, 12500].
+Process-averaged run time: 0.2907813098281622 s
+Run 3
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 12500
+n^2/p = 156250000.0
+ [OK]
+Rank [0/1]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [12500, 12500].
+Process-averaged run time: 0.290579984895885 s
+Run 4
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 12500
+n^2/p = 156250000.0
+ [OK]
+Rank [0/1]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [12500, 12500].
+Process-averaged run time: 0.29034470580518246 s
+Run 5
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 12500
+n^2/p = 156250000.0
+ [OK]
+Rank [0/1]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [12500, 12500].
+Process-averaged run time: 0.2918849550187588 s
+Run 6
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 12500
+n^2/p = 156250000.0
+ [OK]
+Rank [0/1]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [12500, 12500].
+Process-averaged run time: 0.2739923680201173 s
+Run 7
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 12500
+n^2/p = 156250000.0
+ [OK]
+Rank [0/1]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [12500, 12500].
+Process-averaged run time: 0.2719158595427871 s
+Run 8
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 12500
+n^2/p = 156250000.0
+ [OK]
+Rank [0/1]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [12500, 12500].
+Process-averaged run time: 0.27181614004075527 s
+Run 9
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 12500
+n^2/p = 156250000.0
+ [OK]
+Rank [0/1]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [12500, 12500].
+Process-averaged run time: 0.2738722786307335 s
+Run 10
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 12500
+n^2/p = 156250000.0
+ [OK]
+Rank [0/1]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [12500, 12500].
+Process-averaged run time: 0.27227465342730284 s
+Run 11
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 12500
+n^2/p = 156250000.0
+ [OK]
+Rank [0/1]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [12500, 12500].
+Process-averaged run time: 0.27189083211123943 s
+
+============================= JOB FEEDBACK =============================
+
+NodeName=uc2n261
+Job ID: 24673723
+Cluster: uc2
+User/Group: ku4408/scc
+State: COMPLETED (exit code 0)
+Nodes: 1
+Cores per node: 40
+CPU Utilized: 00:00:27
+CPU Efficiency: 0.47% of 01:35:20 core-walltime
+Job Wall-clock time: 00:02:23
+Memory Utilized: 398.27 MB
+Memory Efficiency: 0.89% of 43.95 GB
diff --git a/2_psrs_cdist/solutions/A1/weak_scaling/nodes_16.out b/2_psrs_cdist/solutions/A1/weak_scaling/nodes_16.out
new file mode 100644
index 0000000..296c16a
--- /dev/null
+++ b/2_psrs_cdist/solutions/A1/weak_scaling/nodes_16.out
@@ -0,0 +1,1687 @@
+Run 1
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks in global distance matrix...
+Rank [11/16]: Calculate diagonal blocks in global distance matrix...
+Rank [13/16]: Calculate diagonal blocks in global distance matrix...
+Rank [2/16]: Calculate diagonal blocks in global distance matrix...
+Rank [10/16]: Calculate diagonal blocks in global distance matrix...
+Rank [1/16]: Calculate diagonal blocks in global distance matrix...
+Rank [9/16]: Calculate diagonal blocks in global distance matrix...
+Rank [4/16]: Calculate diagonal blocks in global distance matrix...
+Rank [7/16]: Calculate diagonal blocks in global distance matrix...
+Rank [6/16]: Calculate diagonal blocks in global distance matrix...
+Rank [5/16]: Calculate diagonal blocks in global distance matrix...
+Rank [8/16]: Calculate diagonal blocks in global distance matrix...
+Rank [3/16]: Calculate diagonal blocks in global distance matrix...
+Rank [12/16]: Calculate diagonal blocks in global distance matrix...
+Rank [15/16]: Calculate diagonal blocks in global distance matrix...
+Rank [14/16]: Calculate diagonal blocks in global distance matrix...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [8/16]: Start tile-wise ring communication...
+Rank [12/16]: Start tile-wise ring communication...
+Rank [7/16]: Start tile-wise ring communication...
+Rank [6/16]: Start tile-wise ring communication...
+Rank [11/16]: Start tile-wise ring communication...
+Rank [14/16]: Start tile-wise ring communication...
+Rank [13/16]: Start tile-wise ring communication...
+Rank [9/16]: Start tile-wise ring communication...
+Rank [10/16]: Start tile-wise ring communication...
+Rank [3/16]: Start tile-wise ring communication...
+Rank [2/16]: Start tile-wise ring communication...
+Rank [1/16]: Start tile-wise ring communication...
+Rank [15/16]: Start tile-wise ring communication...
+Rank [4/16]: Start tile-wise ring communication...
+Rank [5/16]: Start tile-wise ring communication...
+Rank [15/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 3.878906511410605 s
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Run 2
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks in global distance matrix...
+Rank [13/16]: Calculate diagonal blocks in global distance matrix...
+Rank [8/16]: Calculate diagonal blocks in global distance matrix...
+Rank [15/16]: Calculate diagonal blocks in global distance matrix...
+Rank [6/16]: Calculate diagonal blocks in global distance matrix...
+Rank [4/16]: Calculate diagonal blocks in global distance matrix...
+Rank [10/16]: Calculate diagonal blocks in global distance matrix...
+Rank [2/16]: Calculate diagonal blocks in global distance matrix...
+Rank [7/16]: Calculate diagonal blocks in global distance matrix...
+Rank [14/16]: Calculate diagonal blocks in global distance matrix...
+Rank [12/16]: Calculate diagonal blocks in global distance matrix...
+Rank [9/16]: Calculate diagonal blocks in global distance matrix...
+Rank [11/16]: Calculate diagonal blocks in global distance matrix...
+Rank [5/16]: Calculate diagonal blocks in global distance matrix...
+Rank [3/16]: Calculate diagonal blocks in global distance matrix...
+Rank [1/16]: Calculate diagonal blocks in global distance matrix...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [14/16]: Start tile-wise ring communication...
+Rank [3/16]: Start tile-wise ring communication...
+Rank [15/16]: Start tile-wise ring communication...
+Rank [2/16]: Start tile-wise ring communication...
+Rank [6/16]: Start tile-wise ring communication...
+Rank [7/16]: Start tile-wise ring communication...
+Rank [5/16]: Start tile-wise ring communication...
+Rank [10/16]: Start tile-wise ring communication...
+Rank [12/16]: Start tile-wise ring communication...
+Rank [13/16]: Start tile-wise ring communication...
+Rank [11/16]: Start tile-wise ring communication...
+Rank [4/16]: Start tile-wise ring communication...
+Rank [8/16]: Start tile-wise ring communication...
+Rank [1/16]: Start tile-wise ring communication...
+Rank [9/16]: Start tile-wise ring communication...
+Rank [15/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.593085914151743 s
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Run 3
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks in global distance matrix...
+Rank [3/16]: Calculate diagonal blocks in global distance matrix...
+Rank [4/16]: Calculate diagonal blocks in global distance matrix...
+Rank [5/16]: Calculate diagonal blocks in global distance matrix...
+Rank [2/16]: Calculate diagonal blocks in global distance matrix...
+Rank [9/16]: Calculate diagonal blocks in global distance matrix...
+Rank [10/16]: Calculate diagonal blocks in global distance matrix...
+Rank [12/16]: Calculate diagonal blocks in global distance matrix...
+Rank [11/16]: Calculate diagonal blocks in global distance matrix...
+Rank [1/16]: Calculate diagonal blocks in global distance matrix...
+Rank [6/16]: Calculate diagonal blocks in global distance matrix...
+Rank [14/16]: Calculate diagonal blocks in global distance matrix...
+Rank [8/16]: Calculate diagonal blocks in global distance matrix...
+Rank [15/16]: Calculate diagonal blocks in global distance matrix...
+Rank [7/16]: Calculate diagonal blocks in global distance matrix...
+Rank [13/16]: Calculate diagonal blocks in global distance matrix...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [4/16]: Start tile-wise ring communication...
+Rank [5/16]: Start tile-wise ring communication...
+Rank [9/16]: Start tile-wise ring communication...
+Rank [10/16]: Start tile-wise ring communication...
+Rank [1/16]: Start tile-wise ring communication...
+Rank [6/16]: Start tile-wise ring communication...
+Rank [3/16]: Start tile-wise ring communication...
+Rank [2/16]: Start tile-wise ring communication...
+Rank [15/16]: Start tile-wise ring communication...
+Rank [12/16]: Start tile-wise ring communication...
+Rank [7/16]: Start tile-wise ring communication...
+Rank [14/16]: Start tile-wise ring communication...
+Rank [13/16]: Start tile-wise ring communication...
+Rank [11/16]: Start tile-wise ring communication...
+Rank [8/16]: Start tile-wise ring communication...
+Rank [15/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.6429246350307949 s
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Run 4
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks in global distance matrix...
+Rank [2/16]: Calculate diagonal blocks in global distance matrix...
+Rank [10/16]: Calculate diagonal blocks in global distance matrix...
+Rank [7/16]: Calculate diagonal blocks in global distance matrix...
+Rank [1/16]: Calculate diagonal blocks in global distance matrix...
+Rank [14/16]: Calculate diagonal blocks in global distance matrix...
+Rank [12/16]: Calculate diagonal blocks in global distance matrix...
+Rank [8/16]: Calculate diagonal blocks in global distance matrix...
+Rank [4/16]: Calculate diagonal blocks in global distance matrix...
+Rank [5/16]: Calculate diagonal blocks in global distance matrix...
+Rank [15/16]: Calculate diagonal blocks in global distance matrix...
+Rank [6/16]: Calculate diagonal blocks in global distance matrix...
+Rank [13/16]: Calculate diagonal blocks in global distance matrix...
+Rank [11/16]: Calculate diagonal blocks in global distance matrix...
+Rank [3/16]: Calculate diagonal blocks in global distance matrix...
+Rank [9/16]: Calculate diagonal blocks in global distance matrix...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [14/16]: Start tile-wise ring communication...
+Rank [15/16]: Start tile-wise ring communication...
+Rank [10/16]: Start tile-wise ring communication...
+Rank [7/16]: Start tile-wise ring communication...
+Rank [2/16]: Start tile-wise ring communication...
+Rank [9/16]: Start tile-wise ring communication...
+Rank [1/16]: Start tile-wise ring communication...
+Rank [3/16]: Start tile-wise ring communication...
+Rank [8/16]: Start tile-wise ring communication...
+Rank [4/16]: Start tile-wise ring communication...
+Rank [5/16]: Start tile-wise ring communication...
+Rank [6/16]: Start tile-wise ring communication...
+Rank [13/16]: Start tile-wise ring communication...
+Rank [11/16]: Start tile-wise ring communication...
+Rank [12/16]: Start tile-wise ring communication...
+Rank [15/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.6121835195808671 s
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Run 5
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks in global distance matrix...
+Rank [8/16]: Calculate diagonal blocks in global distance matrix...
+Rank [15/16]: Calculate diagonal blocks in global distance matrix...
+Rank [2/16]: Calculate diagonal blocks in global distance matrix...
+Rank [9/16]: Calculate diagonal blocks in global distance matrix...
+Rank [4/16]: Calculate diagonal blocks in global distance matrix...
+Rank [3/16]: Calculate diagonal blocks in global distance matrix...
+Rank [12/16]: Calculate diagonal blocks in global distance matrix...
+Rank [7/16]: Calculate diagonal blocks in global distance matrix...
+Rank [1/16]: Calculate diagonal blocks in global distance matrix...
+Rank [13/16]: Calculate diagonal blocks in global distance matrix...
+Rank [11/16]: Calculate diagonal blocks in global distance matrix...
+Rank [14/16]: Calculate diagonal blocks in global distance matrix...
+Rank [10/16]: Calculate diagonal blocks in global distance matrix...
+Rank [5/16]: Calculate diagonal blocks in global distance matrix...
+Rank [6/16]: Calculate diagonal blocks in global distance matrix...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [13/16]: Start tile-wise ring communication...
+Rank [11/16]: Start tile-wise ring communication...
+Rank [14/16]: Start tile-wise ring communication...
+Rank [8/16]: Start tile-wise ring communication...
+Rank [15/16]: Start tile-wise ring communication...
+Rank [5/16]: Start tile-wise ring communication...
+Rank [6/16]: Start tile-wise ring communication...
+Rank [9/16]: Start tile-wise ring communication...
+Rank [4/16]: Start tile-wise ring communication...
+Rank [7/16]: Start tile-wise ring communication...
+Rank [1/16]: Start tile-wise ring communication...
+Rank [10/16]: Start tile-wise ring communication...
+Rank [12/16]: Start tile-wise ring communication...
+Rank [3/16]: Start tile-wise ring communication...
+Rank [2/16]: Start tile-wise ring communication...
+Rank [15/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.61757562198909 s
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Run 6
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks in global distance matrix...
+Rank [10/16]: Calculate diagonal blocks in global distance matrix...
+Rank [15/16]: Calculate diagonal blocks in global distance matrix...
+Rank [2/16]: Calculate diagonal blocks in global distance matrix...
+Rank [8/16]: Calculate diagonal blocks in global distance matrix...
+Rank [6/16]: Calculate diagonal blocks in global distance matrix...
+Rank [13/16]: Calculate diagonal blocks in global distance matrix...
+Rank [1/16]: Calculate diagonal blocks in global distance matrix...
+Rank [3/16]: Calculate diagonal blocks in global distance matrix...
+Rank [5/16]: Calculate diagonal blocks in global distance matrix...
+Rank [7/16]: Calculate diagonal blocks in global distance matrix...
+Rank [4/16]: Calculate diagonal blocks in global distance matrix...
+Rank [14/16]: Calculate diagonal blocks in global distance matrix...
+Rank [9/16]: Calculate diagonal blocks in global distance matrix...
+Rank [11/16]: Calculate diagonal blocks in global distance matrix...
+Rank [12/16]: Calculate diagonal blocks in global distance matrix...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [10/16]: Start tile-wise ring communication...
+Rank [3/16]: Start tile-wise ring communication...
+Rank [5/16]: Start tile-wise ring communication...
+Rank [6/16]: Start tile-wise ring communication...
+Rank [8/16]: Start tile-wise ring communication...
+Rank [9/16]: Start tile-wise ring communication...
+Rank [15/16]: Start tile-wise ring communication...
+Rank [7/16]: Start tile-wise ring communication...
+Rank [2/16]: Start tile-wise ring communication...
+Rank [11/16]: Start tile-wise ring communication...
+Rank [14/16]: Start tile-wise ring communication...
+Rank [12/16]: Start tile-wise ring communication...
+Rank [4/16]: Start tile-wise ring communication...
+Rank [13/16]: Start tile-wise ring communication...
+Rank [1/16]: Start tile-wise ring communication...
+Rank [15/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.615495155681856 s
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Run 7
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks in global distance matrix...
+Rank [15/16]: Calculate diagonal blocks in global distance matrix...
+Rank [13/16]: Calculate diagonal blocks in global distance matrix...
+Rank [7/16]: Calculate diagonal blocks in global distance matrix...
+Rank [4/16]: Calculate diagonal blocks in global distance matrix...
+Rank [11/16]: Calculate diagonal blocks in global distance matrix...
+Rank [3/16]: Calculate diagonal blocks in global distance matrix...
+Rank [10/16]: Calculate diagonal blocks in global distance matrix...
+Rank [2/16]: Calculate diagonal blocks in global distance matrix...
+Rank [6/16]: Calculate diagonal blocks in global distance matrix...
+Rank [1/16]: Calculate diagonal blocks in global distance matrix...
+Rank [8/16]: Calculate diagonal blocks in global distance matrix...
+Rank [14/16]: Calculate diagonal blocks in global distance matrix...
+Rank [5/16]: Calculate diagonal blocks in global distance matrix...
+Rank [12/16]: Calculate diagonal blocks in global distance matrix...
+Rank [9/16]: Calculate diagonal blocks in global distance matrix...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [5/16]: Start tile-wise ring communication...
+Rank [6/16]: Start tile-wise ring communication...
+Rank [15/16]: Start tile-wise ring communication...
+Rank [8/16]: Start tile-wise ring communication...
+Rank [7/16]: Start tile-wise ring communication...
+Rank [12/16]: Start tile-wise ring communication...
+Rank [9/16]: Start tile-wise ring communication...
+Rank [11/16]: Start tile-wise ring communication...
+Rank [2/16]: Start tile-wise ring communication...
+Rank [13/16]: Start tile-wise ring communication...
+Rank [10/16]: Start tile-wise ring communication...
+Rank [3/16]: Start tile-wise ring communication...
+Rank [14/16]: Start tile-wise ring communication...
+Rank [4/16]: Start tile-wise ring communication...
+Rank [1/16]: Start tile-wise ring communication...
+Rank [15/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.600488442578353 s
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Run 8
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks in global distance matrix...
+Rank [3/16]: Calculate diagonal blocks in global distance matrix...
+Rank [8/16]: Calculate diagonal blocks in global distance matrix...
+Rank [6/16]: Calculate diagonal blocks in global distance matrix...
+Rank [9/16]: Calculate diagonal blocks in global distance matrix...
+Rank [14/16]: Calculate diagonal blocks in global distance matrix...
+Rank [5/16]: Calculate diagonal blocks in global distance matrix...
+Rank [4/16]: Calculate diagonal blocks in global distance matrix...
+Rank [1/16]: Calculate diagonal blocks in global distance matrix...
+Rank [10/16]: Calculate diagonal blocks in global distance matrix...
+Rank [12/16]: Calculate diagonal blocks in global distance matrix...
+Rank [15/16]: Calculate diagonal blocks in global distance matrix...
+Rank [13/16]: Calculate diagonal blocks in global distance matrix...
+Rank [11/16]: Calculate diagonal blocks in global distance matrix...
+Rank [7/16]: Calculate diagonal blocks in global distance matrix...
+Rank [2/16]: Calculate diagonal blocks in global distance matrix...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [3/16]: Start tile-wise ring communication...
+Rank [9/16]: Start tile-wise ring communication...
+Rank [10/16]: Start tile-wise ring communication...
+Rank [14/16]: Start tile-wise ring communication...
+Rank [8/16]: Start tile-wise ring communication...
+Rank [15/16]: Start tile-wise ring communication...
+Rank [5/16]: Start tile-wise ring communication...
+Rank [4/16]: Start tile-wise ring communication...
+Rank [6/16]: Start tile-wise ring communication...
+Rank [2/16]: Start tile-wise ring communication...
+Rank [13/16]: Start tile-wise ring communication...
+Rank [11/16]: Start tile-wise ring communication...
+Rank [7/16]: Start tile-wise ring communication...
+Rank [12/16]: Start tile-wise ring communication...
+Rank [1/16]: Start tile-wise ring communication...
+Rank [15/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.5943497718544677 s
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Run 9
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks in global distance matrix...
+Rank [5/16]: Calculate diagonal blocks in global distance matrix...
+Rank [8/16]: Calculate diagonal blocks in global distance matrix...
+Rank [4/16]: Calculate diagonal blocks in global distance matrix...
+Rank [3/16]: Calculate diagonal blocks in global distance matrix...
+Rank [14/16]: Calculate diagonal blocks in global distance matrix...
+Rank [10/16]: Calculate diagonal blocks in global distance matrix...
+Rank [6/16]: Calculate diagonal blocks in global distance matrix...
+Rank [9/16]: Calculate diagonal blocks in global distance matrix...
+Rank [2/16]: Calculate diagonal blocks in global distance matrix...
+Rank [11/16]: Calculate diagonal blocks in global distance matrix...
+Rank [12/16]: Calculate diagonal blocks in global distance matrix...
+Rank [1/16]: Calculate diagonal blocks in global distance matrix...
+Rank [13/16]: Calculate diagonal blocks in global distance matrix...
+Rank [7/16]: Calculate diagonal blocks in global distance matrix...
+Rank [15/16]: Calculate diagonal blocks in global distance matrix...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [3/16]: Start tile-wise ring communication...
+Rank [7/16]: Start tile-wise ring communication...
+Rank [15/16]: Start tile-wise ring communication...
+Rank [5/16]: Start tile-wise ring communication...
+Rank [8/16]: Start tile-wise ring communication...
+Rank [11/16]: Start tile-wise ring communication...
+Rank [1/16]: Start tile-wise ring communication...
+Rank [14/16]: Start tile-wise ring communication...
+Rank [10/16]: Start tile-wise ring communication...
+Rank [6/16]: Start tile-wise ring communication...
+Rank [9/16]: Start tile-wise ring communication...
+Rank [2/16]: Start tile-wise ring communication...
+Rank [4/16]: Start tile-wise ring communication...
+Rank [12/16]: Start tile-wise ring communication...
+Rank [13/16]: Start tile-wise ring communication...
+Rank [15/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.5934264142997563 s
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Run 10
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks in global distance matrix...
+Rank [1/16]: Calculate diagonal blocks in global distance matrix...
+Rank [8/16]: Calculate diagonal blocks in global distance matrix...
+Rank [5/16]: Calculate diagonal blocks in global distance matrix...
+Rank [3/16]: Calculate diagonal blocks in global distance matrix...
+Rank [13/16]: Calculate diagonal blocks in global distance matrix...
+Rank [12/16]: Calculate diagonal blocks in global distance matrix...
+Rank [7/16]: Calculate diagonal blocks in global distance matrix...
+Rank [10/16]: Calculate diagonal blocks in global distance matrix...
+Rank [4/16]: Calculate diagonal blocks in global distance matrix...
+Rank [9/16]: Calculate diagonal blocks in global distance matrix...
+Rank [15/16]: Calculate diagonal blocks in global distance matrix...
+Rank [2/16]: Calculate diagonal blocks in global distance matrix...
+Rank [14/16]: Calculate diagonal blocks in global distance matrix...
+Rank [6/16]: Calculate diagonal blocks in global distance matrix...
+Rank [11/16]: Calculate diagonal blocks in global distance matrix...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [15/16]: Start tile-wise ring communication...
+Rank [5/16]: Start tile-wise ring communication...
+Rank [10/16]: Start tile-wise ring communication...
+Rank [12/16]: Start tile-wise ring communication...
+Rank [4/16]: Start tile-wise ring communication...
+Rank [3/16]: Start tile-wise ring communication...
+Rank [2/16]: Start tile-wise ring communication...
+Rank [13/16]: Start tile-wise ring communication...
+Rank [6/16]: Start tile-wise ring communication...
+Rank [1/16]: Start tile-wise ring communication...
+Rank [11/16]: Start tile-wise ring communication...
+Rank [8/16]: Start tile-wise ring communication...
+Rank [14/16]: Start tile-wise ring communication...
+Rank [7/16]: Start tile-wise ring communication...
+Rank [9/16]: Start tile-wise ring communication...
+Rank [15/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.5964849035954103 s
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Run 11
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks in global distance matrix...
+Rank [4/16]: Calculate diagonal blocks in global distance matrix...
+Rank [2/16]: Calculate diagonal blocks in global distance matrix...
+Rank [9/16]: Calculate diagonal blocks in global distance matrix...
+Rank [10/16]: Calculate diagonal blocks in global distance matrix...
+Rank [7/16]: Calculate diagonal blocks in global distance matrix...
+Rank [5/16]: Calculate diagonal blocks in global distance matrix...
+Rank [15/16]: Calculate diagonal blocks in global distance matrix...
+Rank [6/16]: Calculate diagonal blocks in global distance matrix...
+Rank [13/16]: Calculate diagonal blocks in global distance matrix...
+Rank [11/16]: Calculate diagonal blocks in global distance matrix...
+Rank [12/16]: Calculate diagonal blocks in global distance matrix...
+Rank [8/16]: Calculate diagonal blocks in global distance matrix...
+Rank [14/16]: Calculate diagonal blocks in global distance matrix...
+Rank [1/16]: Calculate diagonal blocks in global distance matrix...
+Rank [3/16]: Calculate diagonal blocks in global distance matrix...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [5/16]: Start tile-wise ring communication...
+Rank [10/16]: Start tile-wise ring communication...
+Rank [6/16]: Start tile-wise ring communication...
+Rank [4/16]: Start tile-wise ring communication...
+Rank [2/16]: Start tile-wise ring communication...
+Rank [15/16]: Start tile-wise ring communication...
+Rank [8/16]: Start tile-wise ring communication...
+Rank [13/16]: Start tile-wise ring communication...
+Rank [11/16]: Start tile-wise ring communication...
+Rank [14/16]: Start tile-wise ring communication...
+Rank [7/16]: Start tile-wise ring communication...
+Rank [12/16]: Start tile-wise ring communication...
+Rank [1/16]: Start tile-wise ring communication...
+Rank [3/16]: Start tile-wise ring communication...
+Rank [9/16]: Start tile-wise ring communication...
+Rank [15/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.6334015409229323 s
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+
+============================= JOB FEEDBACK =============================
+
+NodeName=uc2n[139-143,184-189,194-197,200]
+Job ID: 24673978
+Cluster: uc2
+User/Group: ku4408/scc
+State: COMPLETED (exit code 0)
+Nodes: 16
+Cores per node: 80
+CPU Utilized: 00:08:26
+CPU Efficiency: 0.25% of 2-08:32:00 core-walltime
+Job Wall-clock time: 00:02:39
+Memory Utilized: 1.11 GB
+Memory Efficiency: 0.08% of 1.37 TB
diff --git a/2_psrs_cdist/solutions/A1/weak_scaling/nodes_2.out b/2_psrs_cdist/solutions/A1/weak_scaling/nodes_2.out
new file mode 100644
index 0000000..f5e4efb
--- /dev/null
+++ b/2_psrs_cdist/solutions/A1/weak_scaling/nodes_2.out
@@ -0,0 +1,301 @@
+Run 1
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+ [OK]
+Rank [0/2]: Local data chunk has shape [8839, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/2]: Local data chunk has shape [8839, 18]...
+Overall number of samples is 17678.
+Overall number of samples is 17678.
+Rank [0/2]: Calculate diagonal blocks in global distance matrix...
+Rank [1/2]: Calculate diagonal blocks in global distance matrix...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [8839, 17678].
+Process-averaged run time: 1.9937571994960308 s
+Rank [1/2]: Local distance matrix has shape [8839, 17678].
+Run 2
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+ [OK]
+Rank [0/2]: Local data chunk has shape [8839, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/2]: Local data chunk has shape [8839, 18]...
+Overall number of samples is 17678.
+Overall number of samples is 17678.
+Rank [0/2]: Calculate diagonal blocks in global distance matrix...
+Rank [1/2]: Calculate diagonal blocks in global distance matrix...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [8839, 17678].
+Process-averaged run time: 0.545912426430732 s
+Rank [1/2]: Local distance matrix has shape [8839, 17678].
+Run 3
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+ [OK]
+Rank [1/2]: Local data chunk has shape [8839, 18]...
+ [OK]
+Rank [0/2]: Local data chunk has shape [8839, 18]...
+Start distance calculations...
+Overall number of samples is 17678.
+Overall number of samples is 17678.
+Rank [0/2]: Calculate diagonal blocks in global distance matrix...
+Rank [1/2]: Calculate diagonal blocks in global distance matrix...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [8839, 17678].
+Process-averaged run time: 0.5461812485009432 s
+Rank [1/2]: Local distance matrix has shape [8839, 17678].
+Run 4
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+ [OK]
+Rank [0/2]: Local data chunk has shape [8839, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/2]: Local data chunk has shape [8839, 18]...
+Overall number of samples is 17678.
+Overall number of samples is 17678.
+Rank [0/2]: Calculate diagonal blocks in global distance matrix...
+Rank [1/2]: Calculate diagonal blocks in global distance matrix...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [8839, 17678].
+Process-averaged run time: 0.5449281865730882 s
+Rank [1/2]: Local distance matrix has shape [8839, 17678].
+Run 5
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+ [OK]
+Rank [0/2]: Local data chunk has shape [8839, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/2]: Local data chunk has shape [8839, 18]...
+Overall number of samples is 17678.
+Overall number of samples is 17678.
+Rank [0/2]: Calculate diagonal blocks in global distance matrix...
+Rank [1/2]: Calculate diagonal blocks in global distance matrix...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [8839, 17678].
+Process-averaged run time: 0.545068675186485 s
+Rank [1/2]: Local distance matrix has shape [8839, 17678].
+Run 6
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+ [OK]
+Rank [1/2]: Local data chunk has shape [8839, 18]...
+ [OK]
+Rank [0/2]: Local data chunk has shape [8839, 18]...
+Start distance calculations...
+Overall number of samples is 17678.
+Overall number of samples is 17678.
+Rank [0/2]: Calculate diagonal blocks in global distance matrix...
+Rank [1/2]: Calculate diagonal blocks in global distance matrix...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [8839, 17678].
+Process-averaged run time: 0.5494211222976446 s
+Rank [1/2]: Local distance matrix has shape [8839, 17678].
+Run 7
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+ [OK]
+Rank [0/2]: Local data chunk has shape [8839, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/2]: Local data chunk has shape [8839, 18]...
+Overall number of samples is 17678.
+Overall number of samples is 17678.
+Rank [0/2]: Calculate diagonal blocks in global distance matrix...
+Rank [1/2]: Calculate diagonal blocks in global distance matrix...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [8839, 17678].
+Process-averaged run time: 0.5465278699994087 s
+Rank [1/2]: Local distance matrix has shape [8839, 17678].
+Run 8
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+ [OK]
+Rank [0/2]: Local data chunk has shape [8839, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/2]: Local data chunk has shape [8839, 18]...
+Overall number of samples is 17678.
+Overall number of samples is 17678.
+Rank [0/2]: Calculate diagonal blocks in global distance matrix...
+Rank [1/2]: Calculate diagonal blocks in global distance matrix...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [8839, 17678].
+Process-averaged run time: 0.5520459348335862 s
+Rank [1/2]: Local distance matrix has shape [8839, 17678].
+Run 9
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+ [OK]
+Rank [1/2]: Local data chunk has shape [8839, 18]...
+ [OK]
+Rank [0/2]: Local data chunk has shape [8839, 18]...
+Start distance calculations...
+Overall number of samples is 17678.
+Overall number of samples is 17678.
+Rank [0/2]: Calculate diagonal blocks in global distance matrix...
+Rank [1/2]: Calculate diagonal blocks in global distance matrix...
+Rank [1/2]: Start tile-wise ring communication...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [8839, 17678].
+Process-averaged run time: 0.5425061988644302 s
+Rank [1/2]: Local distance matrix has shape [8839, 17678].
+Run 10
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+ [OK]
+Rank [0/2]: Local data chunk has shape [8839, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/2]: Local data chunk has shape [8839, 18]...
+Overall number of samples is 17678.
+Overall number of samples is 17678.
+Rank [0/2]: Calculate diagonal blocks in global distance matrix...
+Rank [1/2]: Calculate diagonal blocks in global distance matrix...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [8839, 17678].
+Process-averaged run time: 0.5511140790767968 s
+Rank [1/2]: Local distance matrix has shape [8839, 17678].
+Run 11
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+ [OK]
+Rank [0/2]: Local data chunk has shape [8839, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/2]: Local data chunk has shape [8839, 18]...
+Overall number of samples is 17678.
+Overall number of samples is 17678.
+Rank [0/2]: Calculate diagonal blocks in global distance matrix...
+Rank [1/2]: Calculate diagonal blocks in global distance matrix...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [8839, 17678].
+Process-averaged run time: 0.5446672700345516 s
+Rank [1/2]: Local distance matrix has shape [8839, 17678].
+
+============================= JOB FEEDBACK =============================
+
+NodeName=uc2n[143,148]
+Job ID: 24673741
+Cluster: uc2
+User/Group: ku4408/scc
+State: COMPLETED (exit code 0)
+Nodes: 2
+Cores per node: 80
+CPU Utilized: 00:01:00
+CPU Efficiency: 0.34% of 04:53:20 core-walltime
+Job Wall-clock time: 00:01:50
+Memory Utilized: 1.35 GB
+Memory Efficiency: 0.77% of 175.78 GB
diff --git a/2_psrs_cdist/solutions/A1/weak_scaling/nodes_4.out b/2_psrs_cdist/solutions/A1/weak_scaling/nodes_4.out
new file mode 100644
index 0000000..d68a552
--- /dev/null
+++ b/2_psrs_cdist/solutions/A1/weak_scaling/nodes_4.out
@@ -0,0 +1,499 @@
+Run 1
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+ [OK]
+Rank [0/4]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [2/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [3/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [1/4]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Rank [0/4]: Calculate diagonal blocks in global distance matrix...
+Rank [3/4]: Calculate diagonal blocks in global distance matrix...
+Rank [2/4]: Calculate diagonal blocks in global distance matrix...
+Rank [1/4]: Calculate diagonal blocks in global distance matrix...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [2/4]: Start tile-wise ring communication...
+Rank [1/4]: Start tile-wise ring communication...
+Rank [3/4]: Start tile-wise ring communication...
+Rank [3/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [2/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [6250, 25000].
+Process-averaged run time: 3.6626382177346386 s
+Rank [2/4]: Local distance matrix has shape [6250, 25000].
+Rank [1/4]: Local distance matrix has shape [6250, 25000].
+Rank [3/4]: Local distance matrix has shape [6250, 25000].
+Run 2
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+ [OK]
+Rank [2/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [3/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [0/4]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/4]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Rank [0/4]: Calculate diagonal blocks in global distance matrix...
+Rank [2/4]: Calculate diagonal blocks in global distance matrix...
+Rank [1/4]: Calculate diagonal blocks in global distance matrix...
+Rank [3/4]: Calculate diagonal blocks in global distance matrix...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [2/4]: Start tile-wise ring communication...
+Rank [3/4]: Start tile-wise ring communication...
+Rank [1/4]: Start tile-wise ring communication...
+Rank [3/4]: [DONE]
+Rank [2/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [6250, 25000].
+Process-averaged run time: 0.5522811757400632 s
+Rank [2/4]: Local distance matrix has shape [6250, 25000].
+Rank [1/4]: Local distance matrix has shape [6250, 25000].
+Rank [3/4]: Local distance matrix has shape [6250, 25000].
+Run 3
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+ [OK]
+Rank [0/4]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [2/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [1/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [3/4]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Rank [0/4]: Calculate diagonal blocks in global distance matrix...
+Rank [2/4]: Calculate diagonal blocks in global distance matrix...
+Rank [3/4]: Calculate diagonal blocks in global distance matrix...
+Rank [1/4]: Calculate diagonal blocks in global distance matrix...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [2/4]: Start tile-wise ring communication...
+Rank [1/4]: Start tile-wise ring communication...
+Rank [3/4]: Start tile-wise ring communication...
+Rank [3/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [2/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [6250, 25000].
+Process-averaged run time: 0.5517117989948019 s
+Rank [2/4]: Local distance matrix has shape [6250, 25000].
+Rank [3/4]: Local distance matrix has shape [6250, 25000].
+Rank [1/4]: Local distance matrix has shape [6250, 25000].
+Run 4
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+ [OK]
+Rank [0/4]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [3/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [2/4]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Rank [0/4]: Calculate diagonal blocks in global distance matrix...
+Rank [2/4]: Calculate diagonal blocks in global distance matrix...
+Rank [3/4]: Calculate diagonal blocks in global distance matrix...
+Rank [1/4]: Calculate diagonal blocks in global distance matrix...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [3/4]: Start tile-wise ring communication...
+Rank [1/4]: Start tile-wise ring communication...
+Rank [2/4]: Start tile-wise ring communication...
+Rank [3/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [2/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [6250, 25000].
+Process-averaged run time: 0.551495868130587 s
+Rank [2/4]: Local distance matrix has shape [6250, 25000].
+Rank [3/4]: Local distance matrix has shape [6250, 25000].
+Rank [1/4]: Local distance matrix has shape [6250, 25000].
+Run 5
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+ [OK]
+Rank [2/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [3/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [1/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [0/4]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Rank [0/4]: Calculate diagonal blocks in global distance matrix...
+Rank [2/4]: Calculate diagonal blocks in global distance matrix...
+Rank [3/4]: Calculate diagonal blocks in global distance matrix...
+Rank [1/4]: Calculate diagonal blocks in global distance matrix...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [2/4]: Start tile-wise ring communication...
+Rank [3/4]: Start tile-wise ring communication...
+Rank [1/4]: Start tile-wise ring communication...
+Rank [3/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [2/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [6250, 25000].
+Process-averaged run time: 0.5601713114883751 s
+Rank [2/4]: Local distance matrix has shape [6250, 25000].
+Rank [1/4]: Local distance matrix has shape [6250, 25000].
+Rank [3/4]: Local distance matrix has shape [6250, 25000].
+Run 6
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+ [OK]
+Rank [3/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [1/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [0/4]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [2/4]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Rank [0/4]: Calculate diagonal blocks in global distance matrix...
+Rank [2/4]: Calculate diagonal blocks in global distance matrix...
+Rank [3/4]: Calculate diagonal blocks in global distance matrix...
+Rank [1/4]: Calculate diagonal blocks in global distance matrix...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [2/4]: Start tile-wise ring communication...
+Rank [1/4]: Start tile-wise ring communication...
+Rank [3/4]: Start tile-wise ring communication...
+Rank [3/4]: [DONE]
+Rank [2/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [6250, 25000].
+Process-averaged run time: 0.5571753816329874 s
+Rank [2/4]: Local distance matrix has shape [6250, 25000].
+Rank [3/4]: Local distance matrix has shape [6250, 25000].
+Rank [1/4]: Local distance matrix has shape [6250, 25000].
+Run 7
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+ [OK]
+Rank [2/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [1/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [3/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [0/4]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Rank [0/4]: Calculate diagonal blocks in global distance matrix...
+Rank [3/4]: Calculate diagonal blocks in global distance matrix...
+Rank [1/4]: Calculate diagonal blocks in global distance matrix...
+Rank [2/4]: Calculate diagonal blocks in global distance matrix...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [1/4]: Start tile-wise ring communication...
+Rank [2/4]: Start tile-wise ring communication...
+Rank [3/4]: Start tile-wise ring communication...
+Rank [3/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [2/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [6250, 25000].
+Process-averaged run time: 0.5600139261805452 s
+Rank [2/4]: Local distance matrix has shape [6250, 25000].
+Rank [1/4]: Local distance matrix has shape [6250, 25000].
+Rank [3/4]: Local distance matrix has shape [6250, 25000].
+Run 8
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+ [OK]
+Rank [2/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [3/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [1/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [0/4]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Rank [0/4]: Calculate diagonal blocks in global distance matrix...
+Rank [3/4]: Calculate diagonal blocks in global distance matrix...
+Rank [2/4]: Calculate diagonal blocks in global distance matrix...
+Rank [1/4]: Calculate diagonal blocks in global distance matrix...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [3/4]: Start tile-wise ring communication...
+Rank [2/4]: Start tile-wise ring communication...
+Rank [1/4]: Start tile-wise ring communication...
+Rank [3/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [2/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [6250, 25000].
+Process-averaged run time: 0.5613265818101354 s
+Rank [2/4]: Local distance matrix has shape [6250, 25000].
+Rank [1/4]: Local distance matrix has shape [6250, 25000].
+Rank [3/4]: Local distance matrix has shape [6250, 25000].
+Run 9
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+ [OK]
+Rank [3/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [1/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [2/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [0/4]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Rank [0/4]: Calculate diagonal blocks in global distance matrix...
+Rank [3/4]: Calculate diagonal blocks in global distance matrix...
+Rank [2/4]: Calculate diagonal blocks in global distance matrix...
+Rank [1/4]: Calculate diagonal blocks in global distance matrix...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [1/4]: Start tile-wise ring communication...
+Rank [3/4]: Start tile-wise ring communication...
+Rank [2/4]: Start tile-wise ring communication...
+Rank [3/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [2/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [6250, 25000].
+Process-averaged run time: 0.560131796344649 s
+Rank [2/4]: Local distance matrix has shape [6250, 25000].
+Rank [3/4]: Local distance matrix has shape [6250, 25000].
+Rank [1/4]: Local distance matrix has shape [6250, 25000].
+Run 10
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+ [OK]
+Rank [0/4]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [2/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [3/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [1/4]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Rank [0/4]: Calculate diagonal blocks in global distance matrix...
+Rank [2/4]: Calculate diagonal blocks in global distance matrix...
+Rank [3/4]: Calculate diagonal blocks in global distance matrix...
+Rank [1/4]: Calculate diagonal blocks in global distance matrix...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [3/4]: Start tile-wise ring communication...
+Rank [1/4]: Start tile-wise ring communication...
+Rank [2/4]: Start tile-wise ring communication...
+Rank [3/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [2/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [6250, 25000].
+Process-averaged run time: 0.5606894771917723 s
+Rank [2/4]: Local distance matrix has shape [6250, 25000].
+Rank [3/4]: Local distance matrix has shape [6250, 25000].
+Rank [1/4]: Local distance matrix has shape [6250, 25000].
+Run 11
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+ [OK]
+Rank [0/4]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [3/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [1/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [2/4]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Rank [0/4]: Calculate diagonal blocks in global distance matrix...
+Rank [2/4]: Calculate diagonal blocks in global distance matrix...
+Rank [3/4]: Calculate diagonal blocks in global distance matrix...
+Rank [1/4]: Calculate diagonal blocks in global distance matrix...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [3/4]: Start tile-wise ring communication...
+Rank [1/4]: Start tile-wise ring communication...
+Rank [2/4]: Start tile-wise ring communication...
+Rank [3/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [2/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [6250, 25000].
+Process-averaged run time: 0.5518906468059868 s
+Rank [1/4]: Local distance matrix has shape [6250, 25000].
+Rank [2/4]: Local distance matrix has shape [6250, 25000].
+Rank [3/4]: Local distance matrix has shape [6250, 25000].
+
+============================= JOB FEEDBACK =============================
+
+NodeName=uc2n[114-117]
+Job ID: 24673976
+Cluster: uc2
+User/Group: ku4408/scc
+State: COMPLETED (exit code 0)
+Nodes: 4
+Cores per node: 80
+CPU Utilized: 00:02:04
+CPU Efficiency: 0.23% of 14:40:00 core-walltime
+Job Wall-clock time: 00:02:45
+Memory Utilized: 1.06 GB
+Memory Efficiency: 0.30% of 351.56 GB
diff --git a/2_psrs_cdist/solutions/A1/weak_scaling/nodes_8.out b/2_psrs_cdist/solutions/A1/weak_scaling/nodes_8.out
new file mode 100644
index 0000000..79491b3
--- /dev/null
+++ b/2_psrs_cdist/solutions/A1/weak_scaling/nodes_8.out
@@ -0,0 +1,895 @@
+Run 1
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+ [OK]
+Rank [0/8]: Local data chunk has shape [4419, 18]...
+Start distance calculations...
+ [OK]
+Rank [2/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [5/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [4422, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [3/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [4/8]: Local data chunk has shape [4419, 18]...
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Rank [6/8]: Calculate diagonal blocks in global distance matrix...
+Rank [0/8]: Calculate diagonal blocks in global distance matrix...
+Rank [2/8]: Calculate diagonal blocks in global distance matrix...
+Rank [1/8]: Calculate diagonal blocks in global distance matrix...
+Rank [3/8]: Calculate diagonal blocks in global distance matrix...
+Rank [4/8]: Calculate diagonal blocks in global distance matrix...
+Rank [5/8]: Calculate diagonal blocks in global distance matrix...
+Rank [7/8]: Calculate diagonal blocks in global distance matrix...
+Rank [0/8]: Start tile-wise ring communication...
+Rank [7/8]: Start tile-wise ring communication...
+Rank [6/8]: Start tile-wise ring communication...
+Rank [5/8]: Start tile-wise ring communication...
+Rank [1/8]: Start tile-wise ring communication...
+Rank [3/8]: Start tile-wise ring communication...
+Rank [2/8]: Start tile-wise ring communication...
+Rank [4/8]: Start tile-wise ring communication...
+Rank [6/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [7/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [4419, 35355].
+Process-averaged run time: 3.737699876190163 s
+Rank [6/8]: Local distance matrix has shape [4419, 35355].
+Rank [2/8]: Local distance matrix has shape [4419, 35355].
+Rank [4/8]: Local distance matrix has shape [4419, 35355].
+Rank [3/8]: Local distance matrix has shape [4419, 35355].
+Rank [7/8]: Local distance matrix has shape [4422, 35355].
+Rank [1/8]: Local distance matrix has shape [4419, 35355].
+Rank [5/8]: Local distance matrix has shape [4419, 35355].
+Run 2
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+ [OK]
+Rank [6/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [5/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [0/8]: Local data chunk has shape [4419, 18]...
+Start distance calculations...
+ [OK]
+Rank [3/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [4/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [4422, 18]...
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Rank [0/8]: Calculate diagonal blocks in global distance matrix...
+Rank [6/8]: Calculate diagonal blocks in global distance matrix...
+Rank [2/8]: Calculate diagonal blocks in global distance matrix...
+Rank [1/8]: Calculate diagonal blocks in global distance matrix...
+Rank [5/8]: Calculate diagonal blocks in global distance matrix...
+Rank [3/8]: Calculate diagonal blocks in global distance matrix...
+Rank [4/8]: Calculate diagonal blocks in global distance matrix...
+Rank [7/8]: Calculate diagonal blocks in global distance matrix...
+Rank [7/8]: Start tile-wise ring communication...
+Rank [2/8]: Start tile-wise ring communication...
+Rank [5/8]: Start tile-wise ring communication...
+Rank [6/8]: Start tile-wise ring communication...
+Rank [1/8]: Start tile-wise ring communication...
+Rank [0/8]: Start tile-wise ring communication...
+Rank [3/8]: Start tile-wise ring communication...
+Rank [4/8]: Start tile-wise ring communication...
+Rank [2/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [6/8]: [DONE]
+Rank [7/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [4419, 35355].
+Process-averaged run time: 0.5642747584497556 s
+Rank [7/8]: Local distance matrix has shape [4422, 35355].
+Rank [6/8]: Local distance matrix has shape [4419, 35355].
+Rank [2/8]: Local distance matrix has shape [4419, 35355].
+Rank [3/8]: Local distance matrix has shape [4419, 35355].
+Rank [4/8]: Local distance matrix has shape [4419, 35355].
+Rank [1/8]: Local distance matrix has shape [4419, 35355].
+Rank [5/8]: Local distance matrix has shape [4419, 35355].
+Run 3
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+ [OK]
+Rank [4/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [0/8]: Local data chunk has shape [4419, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [3/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [4422, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [5/8]: Local data chunk has shape [4419, 18]...
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Rank [0/8]: Calculate diagonal blocks in global distance matrix...
+Rank [2/8]: Calculate diagonal blocks in global distance matrix...
+Rank [6/8]: Calculate diagonal blocks in global distance matrix...
+Rank [5/8]: Calculate diagonal blocks in global distance matrix...
+Rank [3/8]: Calculate diagonal blocks in global distance matrix...
+Rank [4/8]: Calculate diagonal blocks in global distance matrix...
+Rank [1/8]: Calculate diagonal blocks in global distance matrix...
+Rank [7/8]: Calculate diagonal blocks in global distance matrix...
+Rank [4/8]: Start tile-wise ring communication...
+Rank [7/8]: Start tile-wise ring communication...
+Rank [0/8]: Start tile-wise ring communication...
+Rank [2/8]: Start tile-wise ring communication...
+Rank [5/8]: Start tile-wise ring communication...
+Rank [1/8]: Start tile-wise ring communication...
+Rank [6/8]: Start tile-wise ring communication...
+Rank [3/8]: Start tile-wise ring communication...
+Rank [2/8]: [DONE]
+Rank [6/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [7/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [4419, 35355].
+Process-averaged run time: 0.6045687387231737 s
+Rank [7/8]: Local distance matrix has shape [4422, 35355].
+Rank [5/8]: Local distance matrix has shape [4419, 35355].
+Rank [3/8]: Local distance matrix has shape [4419, 35355].
+Rank [1/8]: Local distance matrix has shape [4419, 35355].
+Rank [6/8]: Local distance matrix has shape [4419, 35355].
+Rank [4/8]: Local distance matrix has shape [4419, 35355].
+Rank [2/8]: Local distance matrix has shape [4419, 35355].
+Run 4
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+ [OK]
+Rank [0/8]: Local data chunk has shape [4419, 18]...
+Start distance calculations...
+ [OK]
+Rank [3/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [5/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [4/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [4422, 18]...
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Rank [0/8]: Calculate diagonal blocks in global distance matrix...
+Rank [2/8]: Calculate diagonal blocks in global distance matrix...
+Rank [3/8]: Calculate diagonal blocks in global distance matrix...
+Rank [4/8]: Calculate diagonal blocks in global distance matrix...
+Rank [7/8]: Calculate diagonal blocks in global distance matrix...
+Rank [6/8]: Calculate diagonal blocks in global distance matrix...
+Rank [1/8]: Calculate diagonal blocks in global distance matrix...
+Rank [5/8]: Calculate diagonal blocks in global distance matrix...
+Rank [1/8]: Start tile-wise ring communication...
+Rank [3/8]: Start tile-wise ring communication...
+Rank [7/8]: Start tile-wise ring communication...
+Rank [2/8]: Start tile-wise ring communication...
+Rank [5/8]: Start tile-wise ring communication...
+Rank [0/8]: Start tile-wise ring communication...
+Rank [6/8]: Start tile-wise ring communication...
+Rank [4/8]: Start tile-wise ring communication...
+Rank [3/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [6/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [7/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [4419, 35355].
+Process-averaged run time: 0.5654191740322858 s
+Rank [7/8]: Local distance matrix has shape [4422, 35355].
+Rank [5/8]: Local distance matrix has shape [4419, 35355].
+Rank [6/8]: Local distance matrix has shape [4419, 35355].
+Rank [3/8]: Local distance matrix has shape [4419, 35355].
+Rank [2/8]: Local distance matrix has shape [4419, 35355].
+Rank [4/8]: Local distance matrix has shape [4419, 35355].
+Rank [1/8]: Local distance matrix has shape [4419, 35355].
+Run 5
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+ [OK]
+Rank [0/8]: Local data chunk has shape [4419, 18]...
+Start distance calculations...
+ [OK]
+Rank [7/8]: Local data chunk has shape [4422, 18]...
+ [OK]
+Rank [4/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [3/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [5/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [4419, 18]...
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Rank [0/8]: Calculate diagonal blocks in global distance matrix...
+Rank [2/8]: Calculate diagonal blocks in global distance matrix...
+Rank [3/8]: Calculate diagonal blocks in global distance matrix...
+Rank [5/8]: Calculate diagonal blocks in global distance matrix...
+Rank [1/8]: Calculate diagonal blocks in global distance matrix...
+Rank [6/8]: Calculate diagonal blocks in global distance matrix...
+Rank [4/8]: Calculate diagonal blocks in global distance matrix...
+Rank [7/8]: Calculate diagonal blocks in global distance matrix...
+Rank [0/8]: Start tile-wise ring communication...
+Rank [7/8]: Start tile-wise ring communication...
+Rank [2/8]: Start tile-wise ring communication...
+Rank [3/8]: Start tile-wise ring communication...
+Rank [1/8]: Start tile-wise ring communication...
+Rank [6/8]: Start tile-wise ring communication...
+Rank [4/8]: Start tile-wise ring communication...
+Rank [5/8]: Start tile-wise ring communication...
+Rank [2/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [6/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [7/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [4419, 35355].
+Process-averaged run time: 0.5544655029661953 s
+Rank [7/8]: Local distance matrix has shape [4422, 35355].
+Rank [1/8]: Local distance matrix has shape [4419, 35355].
+Rank [5/8]: Local distance matrix has shape [4419, 35355].
+Rank [3/8]: Local distance matrix has shape [4419, 35355].
+Rank [6/8]: Local distance matrix has shape [4419, 35355].
+Rank [2/8]: Local distance matrix has shape [4419, 35355].
+Rank [4/8]: Local distance matrix has shape [4419, 35355].
+Run 6
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+ [OK]
+Rank [5/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [4/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [0/8]: Local data chunk has shape [4419, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [4422, 18]...
+ [OK]
+Rank [3/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [4419, 18]...
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Rank [0/8]: Calculate diagonal blocks in global distance matrix...
+Rank [1/8]: Calculate diagonal blocks in global distance matrix...
+Rank [3/8]: Calculate diagonal blocks in global distance matrix...
+Rank [6/8]: Calculate diagonal blocks in global distance matrix...
+Rank [7/8]: Calculate diagonal blocks in global distance matrix...
+Rank [2/8]: Calculate diagonal blocks in global distance matrix...
+Rank [4/8]: Calculate diagonal blocks in global distance matrix...
+Rank [5/8]: Calculate diagonal blocks in global distance matrix...
+Rank [2/8]: Start tile-wise ring communication...
+Rank [7/8]: Start tile-wise ring communication...
+Rank [0/8]: Start tile-wise ring communication...
+Rank [4/8]: Start tile-wise ring communication...
+Rank [1/8]: Start tile-wise ring communication...
+Rank [6/8]: Start tile-wise ring communication...
+Rank [5/8]: Start tile-wise ring communication...
+Rank [3/8]: Start tile-wise ring communication...
+Rank [4/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [6/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [7/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [4419, 35355].
+Process-averaged run time: 0.554839578224346 s
+Rank [7/8]: Local distance matrix has shape [4422, 35355].
+Rank [3/8]: Local distance matrix has shape [4419, 35355].
+Rank [4/8]: Local distance matrix has shape [4419, 35355].
+Rank [1/8]: Local distance matrix has shape [4419, 35355].
+Rank [6/8]: Local distance matrix has shape [4419, 35355].
+Rank [2/8]: Local distance matrix has shape [4419, 35355].
+Rank [5/8]: Local distance matrix has shape [4419, 35355].
+Run 7
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+ [OK]
+Rank [3/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [4422, 18]...
+ [OK]
+Rank [0/8]: Local data chunk has shape [4419, 18]...
+Start distance calculations...
+ [OK]
+Rank [5/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [4/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [4419, 18]...
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Rank [0/8]: Calculate diagonal blocks in global distance matrix...
+Rank [1/8]: Calculate diagonal blocks in global distance matrix...
+Rank [4/8]: Calculate diagonal blocks in global distance matrix...
+Rank [6/8]: Calculate diagonal blocks in global distance matrix...
+Rank [3/8]: Calculate diagonal blocks in global distance matrix...
+Rank [7/8]: Calculate diagonal blocks in global distance matrix...
+Rank [2/8]: Calculate diagonal blocks in global distance matrix...
+Rank [5/8]: Calculate diagonal blocks in global distance matrix...
+Rank [7/8]: Start tile-wise ring communication...
+Rank [0/8]: Start tile-wise ring communication...
+Rank [1/8]: Start tile-wise ring communication...
+Rank [2/8]: Start tile-wise ring communication...
+Rank [5/8]: Start tile-wise ring communication...
+Rank [6/8]: Start tile-wise ring communication...
+Rank [3/8]: Start tile-wise ring communication...
+Rank [4/8]: Start tile-wise ring communication...
+Rank [2/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [6/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [7/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [4419, 35355].
+Process-averaged run time: 0.5531163478735834 s
+Rank [6/8]: Local distance matrix has shape [4419, 35355].
+Rank [3/8]: Local distance matrix has shape [4419, 35355].
+Rank [5/8]: Local distance matrix has shape [4419, 35355].
+Rank [4/8]: Local distance matrix has shape [4419, 35355].
+Rank [7/8]: Local distance matrix has shape [4422, 35355].
+Rank [2/8]: Local distance matrix has shape [4419, 35355].
+Rank [1/8]: Local distance matrix has shape [4419, 35355].
+Run 8
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+ [OK]
+Rank [0/8]: Local data chunk has shape [4419, 18]...
+Start distance calculations...
+ [OK]
+Rank [3/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [4422, 18]...
+ [OK]
+Rank [5/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [4/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [4419, 18]...
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Rank [0/8]: Calculate diagonal blocks in global distance matrix...
+Rank [2/8]: Calculate diagonal blocks in global distance matrix...
+Rank [5/8]: Calculate diagonal blocks in global distance matrix...
+Rank [7/8]: Calculate diagonal blocks in global distance matrix...
+Rank [6/8]: Calculate diagonal blocks in global distance matrix...
+Rank [4/8]: Calculate diagonal blocks in global distance matrix...
+Rank [1/8]: Calculate diagonal blocks in global distance matrix...
+Rank [3/8]: Calculate diagonal blocks in global distance matrix...
+Rank [7/8]: Start tile-wise ring communication...
+Rank [6/8]: Start tile-wise ring communication...
+Rank [4/8]: Start tile-wise ring communication...
+Rank [2/8]: Start tile-wise ring communication...
+Rank [1/8]: Start tile-wise ring communication...
+Rank [0/8]: Start tile-wise ring communication...
+Rank [5/8]: Start tile-wise ring communication...
+Rank [3/8]: Start tile-wise ring communication...
+Rank [6/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [7/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [4419, 35355].
+Process-averaged run time: 0.5550760158803314 s
+Rank [3/8]: Local distance matrix has shape [4419, 35355].
+Rank [2/8]: Local distance matrix has shape [4419, 35355].
+Rank [7/8]: Local distance matrix has shape [4422, 35355].
+Rank [5/8]: Local distance matrix has shape [4419, 35355].
+Rank [1/8]: Local distance matrix has shape [4419, 35355].
+Rank [6/8]: Local distance matrix has shape [4419, 35355].
+Rank [4/8]: Local distance matrix has shape [4419, 35355].
+Run 9
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+ [OK]
+Rank [0/8]: Local data chunk has shape [4419, 18]...
+Start distance calculations...
+ [OK]
+Rank [5/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [4/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [4422, 18]...
+ [OK]
+Rank [3/8]: Local data chunk has shape [4419, 18]...
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Rank [0/8]: Calculate diagonal blocks in global distance matrix...
+Rank [5/8]: Calculate diagonal blocks in global distance matrix...
+Rank [3/8]: Calculate diagonal blocks in global distance matrix...
+Rank [1/8]: Calculate diagonal blocks in global distance matrix...
+Rank [2/8]: Calculate diagonal blocks in global distance matrix...
+Rank [4/8]: Calculate diagonal blocks in global distance matrix...
+Rank [7/8]: Calculate diagonal blocks in global distance matrix...
+Rank [6/8]: Calculate diagonal blocks in global distance matrix...
+Rank [0/8]: Start tile-wise ring communication...
+Rank [1/8]: Start tile-wise ring communication...
+Rank [6/8]: Start tile-wise ring communication...
+Rank [2/8]: Start tile-wise ring communication...
+Rank [7/8]: Start tile-wise ring communication...
+Rank [5/8]: Start tile-wise ring communication...
+Rank [3/8]: Start tile-wise ring communication...
+Rank [4/8]: Start tile-wise ring communication...
+Rank [0/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [6/8]: [DONE]
+Rank [7/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [4419, 35355].
+Process-averaged run time: 0.5531588207231835 s
+Rank [3/8]: Local distance matrix has shape [4419, 35355].
+Rank [5/8]: Local distance matrix has shape [4419, 35355].
+Rank [2/8]: Local distance matrix has shape [4419, 35355].
+Rank [4/8]: Local distance matrix has shape [4419, 35355].
+Rank [7/8]: Local distance matrix has shape [4422, 35355].
+Rank [6/8]: Local distance matrix has shape [4419, 35355].
+Rank [1/8]: Local distance matrix has shape [4419, 35355].
+Run 10
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+ [OK]
+Rank [5/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [0/8]: Local data chunk has shape [4419, 18]...
+Start distance calculations...
+ [OK]
+Rank [4/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [3/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [4422, 18]...
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Rank [0/8]: Calculate diagonal blocks in global distance matrix...
+Rank [1/8]: Calculate diagonal blocks in global distance matrix...
+Rank [3/8]: Calculate diagonal blocks in global distance matrix...
+Rank [4/8]: Calculate diagonal blocks in global distance matrix...
+Rank [6/8]: Calculate diagonal blocks in global distance matrix...
+Rank [2/8]: Calculate diagonal blocks in global distance matrix...
+Rank [5/8]: Calculate diagonal blocks in global distance matrix...
+Rank [7/8]: Calculate diagonal blocks in global distance matrix...
+Rank [2/8]: Start tile-wise ring communication...
+Rank [7/8]: Start tile-wise ring communication...
+Rank [4/8]: Start tile-wise ring communication...
+Rank [0/8]: Start tile-wise ring communication...
+Rank [6/8]: Start tile-wise ring communication...
+Rank [1/8]: Start tile-wise ring communication...
+Rank [3/8]: Start tile-wise ring communication...
+Rank [5/8]: Start tile-wise ring communication...
+Rank [2/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [6/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [7/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [4419, 35355].
+Process-averaged run time: 0.5648037176579237 s
+Rank [7/8]: Local distance matrix has shape [4422, 35355].
+Rank [6/8]: Local distance matrix has shape [4419, 35355].
+Rank [4/8]: Local distance matrix has shape [4419, 35355].
+Rank [2/8]: Local distance matrix has shape [4419, 35355].
+Rank [5/8]: Local distance matrix has shape [4419, 35355].
+Rank [3/8]: Local distance matrix has shape [4419, 35355].
+Rank [1/8]: Local distance matrix has shape [4419, 35355].
+Run 11
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+ [OK]
+Rank [0/8]: Local data chunk has shape [4419, 18]...
+Start distance calculations...
+ [OK]
+Rank [5/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [4/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [3/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [4422, 18]...
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Rank [0/8]: Calculate diagonal blocks in global distance matrix...
+Rank [5/8]: Calculate diagonal blocks in global distance matrix...
+Rank [3/8]: Calculate diagonal blocks in global distance matrix...
+Rank [1/8]: Calculate diagonal blocks in global distance matrix...
+Rank [6/8]: Calculate diagonal blocks in global distance matrix...
+Rank [7/8]: Calculate diagonal blocks in global distance matrix...
+Rank [4/8]: Calculate diagonal blocks in global distance matrix...
+Rank [2/8]: Calculate diagonal blocks in global distance matrix...
+Rank [7/8]: Start tile-wise ring communication...
+Rank [0/8]: Start tile-wise ring communication...
+Rank [5/8]: Start tile-wise ring communication...
+Rank [6/8]: Start tile-wise ring communication...
+Rank [2/8]: Start tile-wise ring communication...
+Rank [1/8]: Start tile-wise ring communication...
+Rank [4/8]: Start tile-wise ring communication...
+Rank [3/8]: Start tile-wise ring communication...
+Rank [2/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [7/8]: [DONE]
+Rank [6/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [4419, 35355].
+Process-averaged run time: 0.5616679729428142 s
+Rank [6/8]: Local distance matrix has shape [4419, 35355].
+Rank [5/8]: Local distance matrix has shape [4419, 35355].
+Rank [3/8]: Local distance matrix has shape [4419, 35355].
+Rank [4/8]: Local distance matrix has shape [4419, 35355].
+Rank [1/8]: Local distance matrix has shape [4419, 35355].
+Rank [2/8]: Local distance matrix has shape [4419, 35355].
+Rank [7/8]: Local distance matrix has shape [4422, 35355].
+
+============================= JOB FEEDBACK =============================
+
+NodeName=uc2n[171,175,178-183]
+Job ID: 24673977
+Cluster: uc2
+User/Group: ku4408/scc
+State: COMPLETED (exit code 0)
+Nodes: 8
+Cores per node: 80
+CPU Utilized: 00:04:06
+CPU Efficiency: 0.26% of 1-02:40:00 core-walltime
+Job Wall-clock time: 00:02:30
+Memory Utilized: 7.40 GB (estimated maximum)
+Memory Efficiency: 1.05% of 703.12 GB (1.10 GB/core)
diff --git a/2_psrs_cdist/solutions/A1/weak_scaling/weak_scaling.pdf b/2_psrs_cdist/solutions/A1/weak_scaling/weak_scaling.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..30c5da6f129800b67218a7907f53a5b0e2cc2082
GIT binary patch
literal 17695
zcmdUX2{={J*LM<f4P{Cyx-yS<zJ?GkBJ-Geo-?~iG)bm1&twP*r81WYnKc+1$WTay
z5=un_zI`rA_`mhN&-Xpg``+5^bM|!BUVH7e*IB=H;?Yu25<!V#VLXKc@N0KqC^!=C
zZtV<{l7b`jj(ga_5d<rum5aLr9HC|9VCM}-g9bWq85x+Jn=RB(d{qY}cQ+y&Lu!C9
z)U<ZCvmwI8mTnb&i3fFwRzy2EcBw$iib%Bca)aZbTNpyu(aP4z%>j;IuJv-a(X}JO
zO~7LnlmIDqzC<`e#TAesznl}6a}~JRDsoWkCjik6QSW63s9Pc*p=0Ome$>kb%m<=h
z2aZs;vvsnPclQNPM1nu!cq|-)N5aiu2m+WC=;;lYSZXQf=H?FWP*AhgjDV_FIHzXk
z=0J3Wqn4kh<m3Y83P&ip00t=7*|^)<K_q(<z3i-9VZLeQ#;WeS#kSW>-5f$mB){^Q
zlj>97g_Wh@Ms*UR2|^DF$ou8Yao?wY#QfMN;*=Xv&fs$VU``ZfZ%9LmoB;yA__9#U
z-NG!?;wA3dN2mGgoFAzV><wbfoSuOzSJ!v6C=HL_pl%<U$Jdsso-UZ2&%9FUW-zKX
z&l^wMxBHs47BAthAGdQ_l6f+J8`)E&%J~-~u8Xa{-M$a6&J)bvT;Q*1kIW6I&|z$~
z5PPNRi7*(I?9FOA@_a#Mdzp!*vqfOcu0^SBSHfdYHN7>PQ93csz%rl`>OmFSi}J+;
z*%T7nuD#t{CWEz@HVMN}Fz?MRWlneQ;R>VrYEU?pdf|cn7a?l&<Io51AI`tgpD~xp
z8GRLa`^1atYL}O`RT_<31o&{>d&TVHA6i+%91bvDR=y+K{=NF8w%S;3a?v3)16Lgt
zt5TP+C~pk!^SXmCqf1`I9_+H(kxlT94rVcu7S|4a+lh)O%cOrK92gN`Q}tS#crIuR
zX?R~^>S)YerLH}aS7N?Cv5HSH@IPck*z0fDns6e%IGFof;~r%bIpLfhuIQI!pIf4;
zYQ-#?Mvr(desi92<xIrT3Xfchm>Ho%osS&xrD@?5>EZ7m-tqWYxSmEF?tFNP?T%3+
zk+#iFJa^9<d{icIUc5&LiVw^8^ufs4-ZQwIzj^nuP*@c|i-u}bd+OORIc!Mi=Dasq
z3cZBDqmK!i{`ud?vB`>D@<NH5ifhDo2Dwl&u;|!lH}?`0X^YazG0hh}LeGQx-P{lJ
z_poqkx4zZV-dC=tCtg+18Lsz=#SO_m+=^xK>CNA&At+DOk!#|=NKg!Ws-rzTk+qM%
z^Gymlx|x%A2OX^2yaXl6Jem{z$TD00zS>jqsuarzmuxzJCbn>O-C@0xD8Bg8b9uuy
zHQj9`Pv6#+|NPoAKYOt%!Z0k)q7DD#j2TR>RMAp5?ksD>?t^skx1x*?sqEZ3d&w!x
zSQaEL&6@Bnn%bM+hd!g7>?JNTmtTt1pq%<B_m1+yr}Nt1#W$a-x&XuXB%Z(j_@*nX
zstLWc!TYbsn86>h<~axAPZ|$~J)1gIJtyTjdOcFKb|_C{w{x!|y#%|I!NEh?c&}4z
z4BTuVhH|y-L%ixnnNOUe2rx;`3Z)w8?!PaduES;eiDo~0rzYDWpCI`k>TGjPEiIP@
zJ)^&ulsVVQ`9HfcDDBphg?u(JyZ6?+s+t!a+l^=5Nz^yoFgi=kaBSweUmiK8ut(<0
zbh<N6T{&KF`+<uAyS6*%c6#Gr{0iRR1D!6<&j+aAE%Se8I7~bT=W{+xzeTA{Sd!O+
zx3x3eWIQY|=!xfWxvd%97Fd@s9?0pOvO|Z=(J~LTd>SFh@oJ1LXDXo;`}mZ6S~>c5
z9zCmulCmYAXHNnlKpuO9TOzD1bGxm9(>J{liV$1oIopw-xQ|lD?e^gA7Ld^j+NyKa
ze|_4l#1>sUTx+Fu*8kh4ylH2rS?U)VdDR`#R6&kA7+CaLlp=77)RKqdt@A&7d1#cD
znEFkc6_1-2H}}OPXk()i7_U0y)~8xiP^&1}Xmek^eDYG4dFl>zW1piQnyQ?simDXP
zx^!&RyRs8P5FADq`?F7!)EcMl7|5T{*Ymt=$NVTK_uM<Df+Aig6@u`Eul2o`rPUvJ
zi5Xg0<(!V#C2Qj+qEPBsa7ohGxlq73*<)P5zAQ<_|I?1RAE};!aXZD+m4}+q-cxrP
z&CM+a3*GUv?-x&{zA0up9o4=q#KI<Ko8j=GeEVyQqWi;R3$q`KeiJ;>u4A5Po?Bfg
z>oYN|d9M8@LxUA7^?XUlfYyh7oYs#s`?^ILB~QAu)R<N&zf{I+q`a}G4a9tiwjzf1
zZ&$3+_>o8J&?p=>)5=imGuC4hjA^8QC*k$7m;L^i<YC&2<aSi=Cq#8Q9<jCat5-#I
zUzXn*w9pk7d|m#kb7wJ`ovY7me!6BuD0zY@#qE(F*Ssh5=;BK|3Hb8)1KT3PoF)*3
z<9@>DwiTouHfGrOC<gnmD_FxeCgfoHrF8Ea#ygkG-Czv+x~}Tk3>xTMJmkP)$b1Bz
zZ1Su=Uan&D13UMvXhTzYZ>XGecAIg-=DT&wvPO@0`R|var)Md;-p``@9&1$T7^!UE
z|M86Rc5xYH!tvMIQfE1?a?E^l5&JNIX0DiHlWnQT6&u`n{iF96J~+?xxTak9zR?$x
z*GSBGS9p-|bK1p>dx6?mU+wc6sZM<^)T0d*Z0APyC(D%^Fn;8?-nU@z!1YLi;fwRR
z21gEGFOPWU^s&Og>TqOHmdDtk;1B67sPUST)3Y6!TsKt?&3-TaGQ0OEceT?36<(+U
z1sMs;r`)o+KspkkV-h8X!>!m5q_Y$%Gyw<hO2ZA#Q*jJ9L;rG?>OZQYt3z(x(wQN!
zlWcn1*RX76-$ndR9<RrB`^`MfXc$u)$JN9er!7OPu~{j(QqPFz6vlJ<e9I58l;pj5
zC74xStoYHpZ%{FWrhT`Z?Vg<Vb3j8!|C*y-4F7l1q$BPzoRQs1zNi<&xLu+8)Secq
zp2Z#l<$*RzO0TPdDBG|p%A|lJISFyR0<09h!_k?}-K>gf`$bqCc8cYmX6ecd<hWze
z`jK^U1DaObJ<H|?8iV==qO?@J8j!SAKUhv|ZqoOQ+Mm8>V9|mGnX<>R?iFQ`iQ=1`
zGU;3Qe<~I2;L74Tos=mpOj~TaZ+pd?nr!CNyxRHKcd0#VJ0tN<E2j5J+}-4R@dE;B
zsVzbNp|f@-A<_-EKYqLFd!wh?SeGH?l`D3e7WV3o6NW_k2b)A2OcVO_3^;U(b1w;M
z)_M*^Q#iw<8Ds-7(yiQOegt=I_d_>!Y`dc}=hZ?pP$eQxE{LIc_SC4`+S5MIiK~H2
z*XI<{xvlR4?GxJ4hRxFWb`cusny!e8lFZYTlkU~EhlhhsEt0*qF&Nr_Uyy0GW)0C;
z+~4`7s4d~P1x5RQ9(K}l)BCdd4C);M_(gYuGoP|zeJsC@xiQ)<#Ppqxs@=1A1bdAR
zv*rP=r>Tl++CtP=$KU}j{BwgaOw+`@H-%>^bBh{;{F<YRHz#y*>Cqh<=6EGN&@*b6
zF<EpngYDWA!NffMuuSPkMJepW`jjZKtDPeH;|qum=^B4QXU>bRuO;)2U17TukX(1d
zwe4JsQ>=&PV59$Q(?=y*0fVWx+a&Y#WLXz{l{%cunzq>9xm84niNNSH2W~ahp7C=#
zQLZ}4lM^M>|NOXhis}#Zlc6uw&~K!tqqa@&9-Lq3Nd6Y_^QMJ`o>}z<+=Gn#wcNvq
z{hfO_ZB;i4)Ykq{=9;>=IX>@<!_r5)UY<4~=s9sqS%!0d6W=oITflWanrH4Yu9KoR
z@TbeqLS{lJ*H<~medz6PeQ!>8YZ;_`;kMwdNjjF;D$b{7saW1(h&nJ0OOA0D>vJoY
zyW$wkk<Rg`KuhtWroMi<1pBz`*GnGg=H31`Z`o*lihGHjj7{Ap6nOqZled<ssxO7Y
zPV;9GyMD6L7gR>-mT<Wi-HoXro(_7K;8RsxbA@BdW0rH;%cXL%HInJ!F@H(1Jk!j(
zRK(ZM+E0d7L|3-x+DAFYu|4ec3LaWafBD}3<ie)TFZ%i$a1DC(S;sZ;f8bh_sv9|q
zmN4rLGlQS`QaUlCrbPs7+pW{SDJ3+T=TiakezfxOf(OL7RKiO?MU60{PlM+l)xJM2
zb3Vac^SXLK2!B|NwWzsn#XwTdmqVTrk*G4GsRzx@xnbK5u%r3D&*{Esd|Yl=Nb9=w
zAftIhNnXxa#~0$x22rxsO$OXtvFEp>bSsQ2BH&j75*sfXjSl_zPI0sTg=ck-GYh#w
zv*}aXNvfxhUt)fAf4Y+PYG~wK#~AzIqisobW<<JabO=jYgNc!yrs3OTEmHkW6Sqq;
z(hQW}Zc(}I{d`Q`_)hFXWyr1h6Pt`qn%>=jZO|*?TDIZ-MYa9fs;4Z;NL%$Rb1B%W
zPO;aOmC=e_m3ZEPX+}by!t2JQ@a#LAr$U^DIIe!OBI~99LQNfLFaOYkPI@y#kbqF!
z?Xr)%WR78G-wDrM33l<3r^^nx7B=ygsBDn8({74EB_-%m;Ohq}RXk3+Z<6o7_QAxK
z%dAnCLE~|Fr(0qOu0QQ?o?KEnefYe-+|x2|#b-go=RQ8{Y;SVBQDiQakX*`lmZ|1T
zM+v6n{3D&|y{~bnf~k&FkIo~5-_mwQ-aDRr>u9EJTCu)a##EOt9h1|Q&u`zHPc%DZ
zWR@TMGfn*&q3osiN&L%!g?SqHizb2_@DO^hUc*Ba<{!>1^(Z+;a?Y(=rmD~KZ902<
zs#ni>+uOyS@Kl_?=bb&dyAb#4{f0b_wF~~#x$_ZI;OCL|>Udo5<$VFn734<onkAI-
zN7O41wbKqnP@bbsuWk+8bMuGGBDZqF9dDe+K+!_fq@2hpz2U-!VqABAS}L;B$D_W_
zQp!i@&ZmKXpF3mo6p>t*#~Vxt@++*FkQnywTM2_l86dTakYtQXR}x`w?I~db!KSh`
z1!0x)qO8iajrYk)Y0qE{o#@pR=?_zeQa`5DrnQ!1zQ~qJczTa*dnN4+>?ZE`9nBk1
zwAvH0asrG0gP1DnsKk<^wt^E_>Yn|mv@|ZE;o;&<2A@1{WmO8Lt*xo|u>usCUYs*3
z2aXtu=y-qMTXU|Ax01OR?L~FI6IWg5qTMQRA$j-BnEH!$`?9wO2dgVxd8*QN#bi7@
zqU2es`uE*2AK`h@h5YC+Rd=<gSf()!DTJ*w+gy}wtb)2_#s_$zGV4pheQC@O#H=~0
zUEl8T&GNUFbGP-CyWYwgJ*w2q5lwdFmf+VR&lhS>FB$gB%}UV|IjGKF_c`EsmpybK
zrl6lJZ;<}E_kPQteR3n7orU}JPSjy#B^T`8T^!~xCOgR$c*y-_V^t|)fU&Cn#SgFh
zK7%^*13EMe4bn#1PrQ?}c8Q80dA`nn>o6U`of@n>=_$1Vhc_?@Q4+|%FS=s1D#T&J
ztV~ek8M!&{42@iSROKQb{Y_*dPu>lOtZFFAKe8pJ!Q=gh*3i0~_~BYqPg2{h>4&#q
z-pD+e?Br0Qsn=!{-D{=7?JQLf!l+XPYh=soWQg6KYSgD*8dahWespH|p+D{=Iqb)9
zr1fLg&vyqWs(hWjE^G8(*vp;QSg^=^upnVbhB}gXP)jfK`%L^}g|b4rn7h8^Wj*bY
z7nu`ce0cYmF-7KIf6OZ*bhpo<>HwJ&t&-z<Hefa|KhdavFpY6qm2}`3c`qHbW2@!6
zQuhp+v)JC+2!6h%0I7@F%A@A!CUp*^%CWGtPmM*{Zk(A`nACAWy-=l4Gv3)w$@0S8
zB2NSFe7~}IMJqtspE9djfHj}SjBM-d_9-iqi^osu-5wSj3Gn&&)W>x~+IB#VBF^Kj
zZ4S+o#|h($CnrBnx)pw7bE`6nmkiI#?K8jHC^&zd@j;*iADd5l{N~OKmF94I(a*1K
zyl9FgTU>$@m}NiEzJ5meW$v@kTtZ3d$9J3bGL`o@DEFa0*HQ}It=Si>XSo4)HZX)y
zxPLfg6}3p3uIdLY<4o+>_lyrjzvl~!eRUO20(sA=$rd*Z(2v#oj1c`lCt|-^O$Ac~
zKB;pZL(3fvNvGpV&}aFcc$zgsWKN$rR8;I#d_~gcD2?#$Lo}H|fn58o-40(<hw%^U
zf6okPio9W$!<g(Xzw=>RoYK|zv$|)@^Ia3PVFY@fFutpp(d&hcBMX#oD{3un_g#}d
z;O;6eyuma#uwOCQe^BVSSQR$~)K-OA846a%HZ_j+nH+;1=|9|O;P+uJ9@_fi?)T%d
z#|u=OV@!2xh?lQk*8Dd9a4NAQ^aG<aV-1fmH?M=(@!*?W47vkxJ6~ZWWg5OtJhble
zZ>pp1xymp7);jKdd*rFRf(fpEDf+F=!WQ$*1>EkRLJ8fX82`xK!d9D9iw1@oZ23A0
ztn>UBZ+t1SO`VMFyPclJ>#AZNHljRKgAJ%U+;4Mue?8aC6lFXshPmSA<x5gS;(2?F
za>Mp#*MGyfg<3~%zHT4>GP{o#Wz683rJ%g|FoV&%^vcZ1Y13*WeJeI+fm5>e&m%)~
zbb2GNTn~wC8o+n=mRcR`wejZ4=PoP0aGsZ&<=XveYbPtO-iA9oIp0fb4!Ks=7aWNV
z6*1v44DP<wWu2wc{P;-Nmps}pEJyUOVOqu?K6)JRK*wY+7a0vDug1dZdWREHvxx3%
zr*>_yL>m~%xPN%Vz~C|Z$W8KQs$3K{@cw~T9nPlc2dz){Jbd`(EcIItwq1s265e>Q
z$vP!p-E#vvk6`)q1{*wn18WoY5BmUvFV;eyX<#ORkvZ7NH?U(d8yI9L4Ei7RaFiT9
zffLAH+F7ox)?G8tv1ZSIxbJUW96Tr95w1Zw(;k%js;K+<g-<5R(jRXXq|9U&ZA%g4
zA0VLbt0Qbav^M#(#|$3V5rMOt;>tW-3*sKec3(+Hi|U7+y3ty%*Q~~9YHH|XsnSHX
z^OEqn1di6eVfOQ1x=Y8h5hoXn;ApiC*tCIVghu{@C84e@=eC8DwrU1;+LDKHGv`#V
z^E1Ma#UrlvO7Gfud7G)7A|nTUd+R<$u!SLx6p9Pg?h~BHW`2xP{ML}zFe*DICGOKw
z#g3mmRNMK$t!X^RTugw*;64BDL>i5QaqnNH+{%A_jzVa^B|&52<7Kp8EL-rgWY?NX
zdL5Mm!gq>WQZ#Se-$5)<S5z7ELcZbkc{Eq}$~v9@!#!b!a$Y5tG{Oz}@m_6Tb^auS
zcWRukY3Geg(5B;r9DU0wnY!n`?(;8d`bW1Nk5tRpfGHbTS163c-&a^VvI;t42>zP)
zC}_vtcSup6Wloy!*E*Jj{YuUgv&U(Be%3|aU@A0kMjt=R$<!OiA9tncrUx=tx_>w(
zr@mRb{A=Q+Yl2eg(PX#IR!%!s2&vLGzrZ|w9ehwXh9-*UJ*WMGXHyXGQ+(9XoSWLs
z?X(r8Lmj8`YLl+L$#0dsRW(s{w`SZ&v}668P^;&(1Z|P<Dule1x82f}qJauQPE$z1
z&e_V~sIHZpw}`yEi!C&3FK;5j(aH;sgdx<dmaoyI%?!{X1v_sWFDDP8yB8d_j9=*<
zwI-6#Gw4AW7(&C!6<|>RaeOd{{1W&if<lTR;UZWt5K00#l~BM>2BA<8f$CO7FDDX0
zC5l8s#~k?c>&grub$~)G0Gt7EiiB#x5Q=U9Yl2|4-*Dba5e%W=WN&W=KrsjfGl4_<
z*W1I&1^{&6J>J{J%G(i+2P>`!7XwN|11=8gb>I@<+8U0qfg=E0z>Mqx4hB@36UabY
zS2)5AhHwW&gZ%^WE^jyjpms+A4hGEN<6s&9^g<kVvL!kK5+NnB8b2ekZpEViy*{g$
zOJc%bIO$^L;0@-o#IXO8r*MRvw+#ddN}vJh&|{z*5j6BxM6mKuwsUfDBti>rxN->q
zM*xL7*~qy$xYz;e5xPV>R|Dum-O86VEjS8`MXp98S2U#w9FIi)zeuqE?l>5%I2?<^
z!*O6O#4+MvZSgP^5{ZV3i=p8dBpy_XkrED#AvM5=BjH%EN>Gl$<Dd@zDZy6PVZlV9
z1U+OW;lLQ+VMw?bct;b%qv2v0ET{w=0WMuAiIb=jgJB?IKoSE|j|UDV2?-o%Ck7YC
zfxZ~f4$8p>5d%J0JP=87;7^9eM4^xvaDf69-~nhH44Mr@HIzeC;K0M7ga-qQ0a3$A
zfc7BA1L?+s0}Z+s1A}3(7*fK6zTg_L2n(J7D)8cB%Z&i5KxZ7F3=2Aw>TqZvMOZi<
zI1usR8V4vN4F(2*azMb+x<M=iM37hu9*PIVk=U|A9h8$q2$~-j^hBZXVx%r$PAlRA
zBdrMFDAb*lfMzE3ggE^B0wNF^0x1C~gK8kE0F^6}07(F8;ef<LImE6NLYAlkAUGb_
zlo0<QN%(_QQXy2a#9WvdGzv%%anLBF^5p@*;y}Gu{(+={BnXf&tVt-OI1Y>j9=t3W
zEBT6KtmM%1S0n?Hia!!m2bH1FFi1jH5=l5foft}jBnXf&tW99SfwX`Hh9qNEf~`o#
zO1>lzOM(GiuTBtgEA@~7tRzS{fTY2o{5R6E+Hs%=)ILcu0+ma`8FV1E3w8%7O2HbA
zUJfa+frD*IDzk-yH6>ly!GVx~D;QAA=;edR365Tlz;K15mt!5=;6US%dbop~zjWmV
z2VeoI3{sy<qbwZ)VAm~`lMV;4GNd~Y=dq-~fqDn4PAWSNwl3-FcXSBYJnP&v693>6
z!G)7h90mU?w;ATd&&&c8=j(!Ns0izT^my2ei=}OP2v^z2j*=@Lx@tNa`NwFTbH&*A
zkMTU3$F}*2?)tpV^9xJGMe<l}?wAmTqz4HOZ!Av8(7NxjUT+tG4Zn`WfAKD2O0fFC
zlT@~eF*xxp?5$EF$&(}#n(%;v>A@dfq{F*5)rQk;V4`C;Fwrr{e{iEq$1XdBWr9Sv
z*M0BJn7Pb<C;8>+em<V@^Lu)CC^{$JQ8Uuzh{nd9J3T(_QJns1ze|U`{g82XQrMgL
z(M}fY$?Oxo50%V}_?q6GvzNRzrY*rrg;&3tc}+O?g{Uum4{wIaSB3-CotHSC;zX&%
zh=+6R1h6(&f1G~o|1%st<g8}qqI1oEuavoh{@57R!LWTcNvxwIa@6C8>K%<k6OJj8
z%T0bNyYEju+5ghHwS=l5xqZKYebIZOj;I@J2q#DDr~K(RInR&Ccgz(D@qAW(9*ptg
zn{;e``?U2{^uo)CEvivB964u(Zn|}gT2|{yDq7oWcojNb(ASvNVt!KDaF=%b7Z*i>
zfOpKZcNPupG)EF$cK9CpPQj7$f>)K^ptnC?W9qt8Z|Ce@;~wJP4VHc7XogIq6?fgg
zyu{$NRoog$fZ5NwP2H2#d`*`^zApCY+-Ke@%TsZi$J&N#GXgay5jrpzi_76X<=}Pr
zr?6(w!TRe%D$D^%!{$C`AM9?nJ3x=wTb*Rkez`f0wp9@?EyaL-&a+(-Sw(sDxqV+)
z4d-wvm(C-l#3wzBpC1cuxvyI>IDVGV_1bsuAB8loKl+$7eEi?3ykQC*O)RXl8)1oS
zGNhE&u2e;xjDF{m#g|phpBd;P;~c7x#P2W^cJCg4C;dd_2ZvYsoQ`HM@?355l@7H$
zsijj_kKOuBf!d)0JGo=wbjRJAo*&yZIQ1AdU>LN^SM0N24EvY8tSJ!-UX&gj50XLZ
z@9dYY&C9F6KmT_6PHc)s^fYJN6DIZ)-xMycBKwwpV_UfWkgi%&?botxOjl=zIQGR~
zps@;$@#robam$EgUGz!lFr=Eo&z(0i-~PN_t^4o}2hn~C8>h5Ji8i-zGp6E*qu29v
z4G$u-XwMNV8sg#JdY3=9MWWx1iO29|T$uXWFYF_6ivB$fK7x~FbWYIkEIu%M;`r3&
zU0!mFW-sPA6syK~;Jc}JbZc8rroB05ao2~)QO?ikEuShU%Eb9-T=2^FRIS3&oJ%^|
z9tARaV}+Tg-&mN6Y{0M8?;9)p!v4!Ul-B-81y0)c5<!S*?oV7E97@+j?575Akf#c0
zshz68aCb5U9Hd3X-yGq&e(6R)?FIRdZm-@4*2Aw6bl~lN+d_#GSMV`RlI&@i16E=-
zk#&kF%CvFf6=^D7i68Z_bKbk{CJdDM!mjR%e20k?$Po0;^xYx+p%rh!7c2c5KCGh?
z)|Q(YfB%5%{dkz9eb*LY#>ydECUx-#RVq5+Z1L!K99Mkyoh+1U&5<0lRc|lkO6h%~
z*`G4E$K#eopRGnh3z6gDTv}OspajdE*^w|jm3l<q*u*T8lORH;irqN<)KSKS7duCH
zGA22kW*f9!{6@79V!Uqy4z4~JmpO<5R?S~y0Hk$P-I`<}l+pCk<LQeX$942yRR3^4
zLeMD?tPVppBTwFsu6(0v#^@Q;cFX*RiPV)7q9(>E$3v?wMDWM}jBJv#!i*?dEY5!L
z6Ae0DP$B=()vl~L`NSZqXpQMCn);Ix3nPd#_j-!xZVyYN<%+o93dlFRX|ts=$}w1t
z#<1jH*2*-hAJm?FKZ~Ytb`Q^`vDx#Px{N~JRk?BVJK0SN6Y+Z2qaz)-nQB>HI9Tto
z9PKJ0yTPWXkmx_r5?aunA6~7{D5I(Voj*}c#Gu(tm(Aj5eE8E7L<+$I?5Froo%*Gn
zG=^cv8NW*WRGu5{cHs<`bnE^U_GT}p3C-AUsVZTal{9j2sNd))!wvSz+wH`Fy$RhO
zYGv<FwSLiDAod$SI$G_uHCy(Ly4-XX{SBq!#zwSEdW3*qws6s|)M_QnJ*({u-Lnm?
z<h$m*s<&hFj_fNl+;_3uS|Q@Ri?{UME!}EUO%mM=O!Y2P>D^f+7>pq2*}gl$&hKS?
zN^533Dl!jX#<E>ESkn!B_~?Jx73$h5=gEQE9cB40%V{20okvI^j4t-vrw@|YLVYtf
zE-~4VW*h3<USa9E7*21o`Q4|od#jB6V%vHblSlXIYR~E9i(1Q@h3@XiVwjfBI>KmF
zeD|Y-a88PM4b|SD+gmVMWHu*N(3Sab6F8YPXpiopH@;kd`-pYJE8{Myhyk}BO7?cV
z=jftr<p*}235CDkT(VO*9*<{28?}>vplhQuw(Kd`t$pFFP<%HvG25W=>cgSZx7|Ks
z1d&fe`6oYqcQi<{4C+$-V(4oTB$Hf_;xid5?>DTzf8Rq6*k$y75utMb#&>Q3-w!;{
znVxT(onybb?a@!uwDlYWcI`Sn3@rS=ZVE*ZeNqSCx@A#Mjv5B*)8vf<$5zC4^a;yR
z`pI(?G!@5{_EIG9nzI;E9kLnFijuuM>?P7RK(Wn(?{r_MURvUlV@@$D`T_%&8ILd0
zJMc#5AMdhBpQW++!L<Qa11zX@R6>r0zlHB$(6_XJRfQ#E4Bo-Jg3l#xHxOl26Q+8&
zxhv5FwlCve0_CPk2RgnDn6rVs`7cpJiofhl%PpsPzshLG*T1Z#+s@;5o9~6RDLZwF
zfE%WPK$$@+9Bk$p@Hr7lV|q|4GJPV)>RHXZEfHU?=*&!Qi_e%Sd=>njY_t3>`wd9@
z@7kI8GQbfe7D*MW6fBz<6DEJ2lT~?BBgq?l#)wjOSGG9S4c-u;Kq0zK!)0{I8$5a=
zI}^NJ{B<>OK#@Y>8XW?+Wpk%B+h?v`y-))bDbrmRHKgT;0K(|8YRyxg{@jMwCVOXj
zisfHMeMo(%(=@xIl(|FX=D`kzTOzP6+Dy`hDF~K*Pdh%3N#=VT$_zNX^%nE)>OhrE
zfzESO9Jv^{)~IZAl{H$2{4Do#Zzs>KVP3ppr!QA$4DwySob_4SDCNOmoN8_-ySm00
zS=0R{l}lzS-!7#jA5w@-dVjK5{&R3$tdz__RNI*ee3)xC1@9<FF3-b8X33(K&DHj)
z&-BME>p4OM!VlAw=5x4wGGR)%%zk}@trz+70aDjGYq*Hw>04{Nvy*Q|a5w8iKWN99
zHF!)WK03Cw%W+}%j{8G4EU{+}<`py2If>VW)@7KBH<`~cNzm7Y+N7rn)7m)l=J`d2
z)pcf+=2|tka2S7?*`_ysB4mSw*udySffua5E`(x~D*YCc=R7;Iu4V72Cwe@{d-21W
zbT=j&=^(v;%ClM0qDRQnS@v?WRRGU<Vx*#xjGmv4!M3)ckizB%oCBAHr_ZVg26Jj%
zNZu7AV&``2Oh2Bkq$EK%H?rtL4l?R%nF|YtMYhSlqp=498wC*qBX`(abGxfvW?!f|
zeVGhJy}@iYum~~0|Ml0|;B<b)P{Gf*^-uM}m1BO^(iQYFlY0_%;`bZt`8)^-(kq`J
zek)7pV-$OR%r2;ry6qa{2c2N${AvTvXHzNYo40TdTUvE;>tA1f?b&wrKtpaGP0P26
z;+t<;*`qiNS-7$k>d;M%W4jXDCyVS&P9-|kMfsjf6Y3UeF{IGh{X+c1`Pr=FA|;fe
zcT*D2!`KbNcAS#?c<Ero$*(*04B=fL_NaJ-hiS!}@d@Ztrj_s)_R-$^8N-ia8tdDw
zI3O3m6Pqm2bcgtPaIUlX(!%p_v06#NaT8-Bo5wP{Kg2f+qsRRWKNDxVTLNjYQ`tX9
zpV2a6?IL2Esjl})1ZSsc8~ENL#L+!MwQ$bRXEd-;ANnrETpPv1#2()|NKcn_+JYk|
zJPrSH^V{qLt}ImauSy%tD+5>#u^x$?B`Z)H3m9L#`LVpFr)Mwwhcg?5BLLsfIt7gW
zm-ku`Z51zaxvf<lLZ8ps=MCB<35C2~?AdysijwYvEYbMYrSMF+@Zst=G1nX$1~I6!
zgA^gBgnC>JU(4#jRcy3ySw0{AK65<xOs0Mq*Ih(wU%2gOxWK?z(=dw7;Z~2`)9n*x
zH>FeL+6qk@d6-z$o>_4@&>u)ai<Wsyi@`^g43n&)-3)eN7$dPk%A7%0FJ5lnBF7kh
zLn#99#rz`|5k(BvkH|+&c#F7Nbf}9T>6|bu{Pa4W!^~5Zj}=k4=kpglIjgo);U`(;
zx_P&4U%X4=27OgW+jl$}@qg6zsb0^r>0*zYEdJT6!RpYFAK^DgN5B0fx3Dw`-9W%M
zFhEhjVf9x5SC3W!Ko<zuN~>qhKZmIsm8!jZ$8oCmI)UnFz|Yrk4=JfmPi|47?O1rE
z(8WC?b)|5dS6$+ZZMS6_QF1S`MGTXCT6!Ncde@^-n=6#sdZn`KwzZgUWwcG&TT=($
zXBPF%V8$r!bH}ayFQV?0*;`0w*M&`NZ7L!=d}jVmOMq(qH}fa2lLUEem_j}UyxXUP
z-=DR;TyMhb>iN<a7lk}nKL6|@On#s!=aL%SVA>nmXu$vX*J<OlCFsdHX(wh`7G$}@
z5x$z){ErrDABA@raNIKuE{o?(`J@w<&-q4I#xF%Q?an~YLuc%0Vws5dncEfBi0iR+
zVfLYc_neY%lch?L5qi4vwF#^oolkRh`HKd&$eh3Bw(W&Yc|cMH=V%z*nx9Qr?oojv
z(+<}TzRS2;^9#FQCVhC(s+-%KxHEu~%s_lk&sfiNg0v=Y7m)@1^cwELPHHB_eNx{d
zuRLRVZ;XA6W_;SrQ0Povy?-95v1t>N=Y!3*nTO&_UpKlJe=emw{C$Be^Ppbhdj0?v
z_j_yoMk?T|kR*9^ZEXdERY(%Vyk8&^a5$_2k(RKERX7K5=aT$se}YH=gdu@Q>xN#7
zbpU6f2o{flqr`w&h(d$o1z<ye$B)35PQUOYEXf!6e}x}Gu?HS@UQX_|aNs~i_}O{6
z!x8Ro03#wE16M>xFFOb?0$*K$f1P~5wKu@B03d-_0$PCc86gLJ{4j(HTml%1R>1fF
z8)yP<VK~ACa)&QNN`GKUz}pHj?6=#I#K9F`#qSkb#p6|2(mI9$Zv6)sX+zfj|A3J&
zz;U{2IxBkUe*laGphYYIAF<%$IFgqegNI2#pbYToqk%ge1LCou1VI2$JqEM~F?UNj
za1TQe!GDopD|Hyq7D`KQ{v{*{2VB%73<56>;R6tA0etxY4Z;Jz9S#C^pafj&XbBt)
zBmj;8pa%kgz-|x&QYi+x=^>y9%AtDT!^VRz2E{;sXiN!lfD4E(l|eaaG8lkLfeAr`
zte_)!5N!qZ#es=0jRD$%=aR00ha3%{11RCdBuKdgI5Y4NrUCGXr8W>WgCb#Z7zwB!
z7C;UVP60s=U@lOC03QfXS!NoE*?_4e27w$f76v^N2m1U0%s@n<Ao+ow1Oag*M41E|
zLQ{i=Sn5t%6R0~#Fx>K*{TgcN0to!l5CCl;VMq{FfO2SAAu(84u9X~63tQn5X#_05
z<iNC-WC6&*uklv|gd|L0B<TLvC=gLdB#czIJb(lMqgN6n2um^m$-$qA#A_fIkW8#f
zBzb^jVkMDeWmzU5d03qwUX$*@7?5nNBxvS%Bo3@0pcRsV4Uz;v*+_B$4ov*2v?3EL
zIm8sO?l4jgU9U={wO*=++Wbn8TtKS|<-gz~lCECSb#ibL;sQV;6@b7k9V1q763zh1
zYycd(R0fpvGH64>`+@d^E$d;x64K#A($T;%vvdQgZIa^wjN$?WW~mrb??41ey_a=A
zZUrrZl>0BN2t-Pd##uTAz^+`n0|x`iOa+4;1@Pq3K%|ocxAfpuDiBnzOi~Gen#-V;
z5+t9b(SA2w{jw{ES{?fE>l<VQK@N06*jT|)keWrvq5oqu;Nbq<4D%nGAw|If1sX!R
zjLa(Cz3PqcH;JoWoPHOsHrUmnJxrPxLLF?HB}V>kN~&97$I|(>^1U_Um(`%;?gaue
ze#bz-L2L=wf6!lG5G?ys@mOF!{N<A>j8^L_wAS>iSC4UzG5Vc2{Z<OEdlP#^^Ofn7
z1WH253^(1Q>Y~)D5ml4zVK!>qv~M$xJMhATFq%z5J6PVWx1UJU`Cm)@Y7f&A>z6)^
zN2og4LV+|uM?lgI#pAdi1ra4A30wZC4IG-k#h3uT{)S?P_(5Echl@MW#mO4(gAtVw
zMGM0ni9`=?Nd&_6SCy!{mxBNd0pfdXkJ^Bb$yfLCu(gL<TiF}|Pgx!Tv?LAh<nE>b
zeRR*SAc;nz!I^^s35Udq3n9^bNF<-YDoOSAva^RliVX(h2$p}~2*m+~V-NpT1}rdA
z9o%iX424I5qi#(Z4hMnF(C>FUa1sO7tSJ+NM0sr)P<iXhAQkdQJ8%PR#<gV_2+;ns
z9U29q@Yj?f(ZHZwQ-(%C%kxKH6ga`xwnO10#MhP~#h`Qa&%Q`8D5_>n8D1Qmvun!4
zAv0l385#`*ctO8k{6nHKkm6iZh6ACBYs<vf&kciu0s{W*3j!I|mx%!<?V5I2<hpeO
zK{3GITiXu1{uv-bW?ef#^4c{=;xJH1!k_fvaO>6siIZ5*D?DlgKBGWD%bM}<kU6oo
z42tmhvkZlWz5`rahJ@ZRpx>|c0H4o50UCdn;n3^k1t~5D-emr4hZL6pK~QVTBtYoP
zx-#r~8JB=e-aq>Ss|uKyYs)a}cmQ6bKuFG-b|@4urPr20rsdi)$d+4MhF`blAYu_R
z8=&8BUZDY3*OftmNNd}P0UL2m8D>3SfX@_|Woz2u(W}>p=w;>PV&_Hrk4$u({Oo{U
s1CccD?vO$zSsyBH_U^!%Bi(?L)thMLMI@;}6dr(#FdiO7O(od>0a*=f1^@s6
literal 0
HcmV?d00001
diff --git a/2_psrs_cdist/solutions/A1/weak_scaling_symmetric/nodes_1.out b/2_psrs_cdist/solutions/A1/weak_scaling_symmetric/nodes_1.out
new file mode 100644
index 0000000..3fbddb4
--- /dev/null
+++ b/2_psrs_cdist/solutions/A1/weak_scaling_symmetric/nodes_1.out
@@ -0,0 +1,158 @@
+Run 1
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 12500
+n^2/p = 156250000.0
+ [OK]
+Rank [0/1]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [12500, 12500].
+Process-averaged run time: 1.9207635009661317 s
+Run 2
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 12500
+n^2/p = 156250000.0
+ [OK]
+Rank [0/1]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [12500, 12500].
+Process-averaged run time: 0.2704510986804962 s
+Run 3
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 12500
+n^2/p = 156250000.0
+ [OK]
+Rank [0/1]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [12500, 12500].
+Process-averaged run time: 0.27157120779156685 s
+Run 4
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 12500
+n^2/p = 156250000.0
+ [OK]
+Rank [0/1]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [12500, 12500].
+Process-averaged run time: 0.2716386690735817 s
+Run 5
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 12500
+n^2/p = 156250000.0
+ [OK]
+Rank [0/1]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [12500, 12500].
+Process-averaged run time: 0.270675303414464 s
+Run 6
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 12500
+n^2/p = 156250000.0
+ [OK]
+Rank [0/1]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [12500, 12500].
+Process-averaged run time: 0.27217359840869904 s
+Run 7
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 12500
+n^2/p = 156250000.0
+ [OK]
+Rank [0/1]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [12500, 12500].
+Process-averaged run time: 0.27377386949956417 s
+Run 8
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 12500
+n^2/p = 156250000.0
+ [OK]
+Rank [0/1]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [12500, 12500].
+Process-averaged run time: 0.2724360143765807 s
+Run 9
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 12500
+n^2/p = 156250000.0
+ [OK]
+Rank [0/1]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [12500, 12500].
+Process-averaged run time: 0.2723913500085473 s
+Run 10
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 12500
+n^2/p = 156250000.0
+ [OK]
+Rank [0/1]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [12500, 12500].
+Process-averaged run time: 0.2717280266806483 s
+Run 11
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 1.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 12500
+n^2/p = 156250000.0
+ [OK]
+Rank [0/1]: Local data chunk has shape [12500, 18]...
+Start distance calculations...
+Rank [0/1]: Local distance matrix has shape [12500, 12500].
+Process-averaged run time: 0.27235382702201605 s
+
+============================= JOB FEEDBACK =============================
+
+NodeName=uc2n261
+Job ID: 24894093
+Cluster: uc2
+User/Group: ku4408/scc
+State: COMPLETED (exit code 0)
+Nodes: 1
+Cores per node: 40
+CPU Utilized: 00:00:26
+CPU Efficiency: 0.83% of 00:52:00 core-walltime
+Job Wall-clock time: 00:01:18
+Memory Utilized: 1.06 GB
+Memory Efficiency: 2.41% of 43.95 GB
diff --git a/2_psrs_cdist/solutions/A1/weak_scaling_symmetric/nodes_16.out b/2_psrs_cdist/solutions/A1/weak_scaling_symmetric/nodes_16.out
new file mode 100644
index 0000000..fc01756
--- /dev/null
+++ b/2_psrs_cdist/solutions/A1/weak_scaling_symmetric/nodes_16.out
@@ -0,0 +1,3095 @@
+Run 1
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks...
+Rank [1/16]: Calculate diagonal blocks...
+Rank [7/16]: Calculate diagonal blocks...
+Rank [9/16]: Calculate diagonal blocks...
+Rank [2/16]: Calculate diagonal blocks...
+Rank [14/16]: Calculate diagonal blocks...
+Rank [10/16]: Calculate diagonal blocks...
+Rank [8/16]: Calculate diagonal blocks...
+Rank [12/16]: Calculate diagonal blocks...
+Rank [15/16]: Calculate diagonal blocks...
+Rank [5/16]: Calculate diagonal blocks...
+Rank [4/16]: Calculate diagonal blocks...
+Rank [11/16]: Calculate diagonal blocks...
+Rank [13/16]: Calculate diagonal blocks...
+Rank [3/16]: Calculate diagonal blocks...
+Rank [6/16]: Calculate diagonal blocks...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [0/16]: Starting iteration 1
+Rank [3/16]: Start tile-wise ring communication...
+Rank [3/16]: Starting iteration 1
+Rank [14/16]: Start tile-wise ring communication...
+Rank [14/16]: Starting iteration 1
+Rank [2/16]: Start tile-wise ring communication...
+Rank [2/16]: Starting iteration 1
+Rank [1/16]: Start tile-wise ring communication...
+Rank [1/16]: Starting iteration 1
+Rank [15/16]: Start tile-wise ring communication...
+Rank [15/16]: Starting iteration 1
+Rank [11/16]: Start tile-wise ring communication...
+Rank [11/16]: Starting iteration 1
+Rank [9/16]: Start tile-wise ring communication...
+Rank [9/16]: Starting iteration 1
+Rank [4/16]: Start tile-wise ring communication...
+Rank [4/16]: Starting iteration 1
+Rank [8/16]: Start tile-wise ring communication...
+Rank [8/16]: Starting iteration 1
+Rank [13/16]: Start tile-wise ring communication...
+Rank [13/16]: Starting iteration 1
+Rank [7/16]: Start tile-wise ring communication...
+Rank [7/16]: Starting iteration 1
+Rank [5/16]: Start tile-wise ring communication...
+Rank [5/16]: Starting iteration 1
+Rank [12/16]: Start tile-wise ring communication...
+Rank [12/16]: Starting iteration 1
+Rank [10/16]: Start tile-wise ring communication...
+Rank [10/16]: Starting iteration 1
+Rank [6/16]: Start tile-wise ring communication...
+Rank [6/16]: Starting iteration 1
+Rank [15/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 2
+Rank [13/16]: Starting iteration 2
+Rank [12/16]: Starting iteration 2
+Rank [11/16]: Starting iteration 2
+Rank [10/16]: Starting iteration 2
+Rank [9/16]: Starting iteration 2
+Rank [8/16]: Starting iteration 2
+Rank [7/16]: Starting iteration 2
+Rank [6/16]: Starting iteration 2
+Rank [5/16]: Starting iteration 2
+Rank [4/16]: Starting iteration 2
+Rank [3/16]: Starting iteration 2
+Rank [2/16]: Starting iteration 2
+Rank [1/16]: Starting iteration 2
+Rank [0/16]: Starting iteration 2
+Rank [15/16]: Starting iteration 3
+Rank [14/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 3
+Rank [12/16]: Starting iteration 3
+Rank [11/16]: Starting iteration 3
+Rank [10/16]: Starting iteration 3
+Rank [9/16]: Starting iteration 3
+Rank [8/16]: Starting iteration 3
+Rank [7/16]: Starting iteration 3
+Rank [6/16]: Starting iteration 3
+Rank [5/16]: Starting iteration 3
+Rank [4/16]: Starting iteration 3
+Rank [1/16]: Starting iteration 3
+Rank [3/16]: Starting iteration 3
+Rank [0/16]: Starting iteration 3
+Rank [2/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 4
+Rank [14/16]: Starting iteration 4
+Rank [15/16]: Starting iteration 4
+Rank [10/16]: Starting iteration 4
+Rank [11/16]: Starting iteration 4
+Rank [12/16]: Starting iteration 4
+Rank [7/16]: Starting iteration 4
+Rank [8/16]: Starting iteration 4
+Rank [9/16]: Starting iteration 4
+Rank [1/16]: Starting iteration 4
+Rank [4/16]: Starting iteration 4
+Rank [2/16]: Starting iteration 4
+Rank [5/16]: Starting iteration 4
+Rank [6/16]: Starting iteration 4
+Rank [0/16]: Starting iteration 4
+Rank [3/16]: Starting iteration 4
+Rank [13/16]: Starting iteration 5
+Rank [14/16]: Starting iteration 5
+Rank [10/16]: Starting iteration 5
+Rank [15/16]: Starting iteration 5
+Rank [12/16]: Starting iteration 5
+Rank [9/16]: Starting iteration 5
+Rank [1/16]: Starting iteration 5
+Rank [5/16]: Starting iteration 5
+Rank [8/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 5
+Rank [2/16]: Starting iteration 5
+Rank [6/16]: Starting iteration 5
+Rank [0/16]: Starting iteration 5
+Rank [4/16]: Starting iteration 5
+Rank [7/16]: Starting iteration 5
+Rank [3/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 6
+Rank [13/16]: Starting iteration 6
+Rank [15/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 6
+Rank [1/16]: Starting iteration 6
+Rank [2/16]: Starting iteration 6
+Rank [10/16]: Starting iteration 6
+Rank [6/16]: Starting iteration 6
+Rank [7/16]: Starting iteration 6
+Rank [5/16]: Starting iteration 6
+Rank [0/16]: Starting iteration 6
+Rank [8/16]: Starting iteration 6
+Rank [14/16]: Starting iteration 6
+Rank [3/16]: Starting iteration 6
+Rank [4/16]: Starting iteration 6
+Rank [9/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 7
+Rank [13/16]: Starting iteration 7
+Rank [0/16]: Starting iteration 7
+Rank [5/16]: Starting iteration 7
+Rank [11/16]: Starting iteration 7
+Rank [6/16]: Starting iteration 7
+Rank [15/16]: Starting iteration 7
+Rank [4/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 7
+Rank [10/16]: Starting iteration 7
+Rank [1/16]: Starting iteration 7
+Rank [7/16]: Starting iteration 7
+Rank [3/16]: Starting iteration 7
+Rank [9/16]: Starting iteration 7
+Rank [2/16]: Starting iteration 7
+Rank [8/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 8
+Rank [0/16]: Starting iteration 8
+Rank [7/16]: Starting iteration 8
+Rank [15/16]: Starting iteration 8
+Rank [1/16]: Starting iteration 8
+Rank [8/16]: Starting iteration 8
+Rank [13/16]: Starting iteration 8
+Rank [6/16]: Starting iteration 8
+Rank [5/16]: Starting iteration 8
+Rank [10/16]: Starting iteration 8
+Rank [3/16]: Starting iteration 8
+Rank [12/16]: Starting iteration 8
+Rank [2/16]: Starting iteration 8
+Rank [4/16]: Starting iteration 8
+Rank [11/16]: Starting iteration 8
+Rank [9/16]: Starting iteration 8
+Rank [15/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 3.8300749802729115 s
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Run 2
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks...
+Rank [6/16]: Calculate diagonal blocks...
+Rank [2/16]: Calculate diagonal blocks...
+Rank [3/16]: Calculate diagonal blocks...
+Rank [5/16]: Calculate diagonal blocks...
+Rank [11/16]: Calculate diagonal blocks...
+Rank [8/16]: Calculate diagonal blocks...
+Rank [14/16]: Calculate diagonal blocks...
+Rank [10/16]: Calculate diagonal blocks...
+Rank [9/16]: Calculate diagonal blocks...
+Rank [4/16]: Calculate diagonal blocks...
+Rank [12/16]: Calculate diagonal blocks...
+Rank [15/16]: Calculate diagonal blocks...
+Rank [7/16]: Calculate diagonal blocks...
+Rank [1/16]: Calculate diagonal blocks...
+Rank [13/16]: Calculate diagonal blocks...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [0/16]: Starting iteration 1
+Rank [7/16]: Start tile-wise ring communication...
+Rank [7/16]: Starting iteration 1
+Rank [9/16]: Start tile-wise ring communication...
+Rank [9/16]: Starting iteration 1
+Rank [11/16]: Start tile-wise ring communication...
+Rank [11/16]: Starting iteration 1
+Rank [15/16]: Start tile-wise ring communication...
+Rank [15/16]: Starting iteration 1
+Rank [13/16]: Start tile-wise ring communication...
+Rank [13/16]: Starting iteration 1
+Rank [5/16]: Start tile-wise ring communication...
+Rank [5/16]: Starting iteration 1
+Rank [8/16]: Start tile-wise ring communication...
+Rank [8/16]: Starting iteration 1
+Rank [10/16]: Start tile-wise ring communication...
+Rank [10/16]: Starting iteration 1
+Rank [6/16]: Start tile-wise ring communication...
+Rank [6/16]: Starting iteration 1
+Rank [2/16]: Start tile-wise ring communication...
+Rank [2/16]: Starting iteration 1
+Rank [1/16]: Start tile-wise ring communication...
+Rank [1/16]: Starting iteration 1
+Rank [3/16]: Start tile-wise ring communication...
+Rank [3/16]: Starting iteration 1
+Rank [4/16]: Start tile-wise ring communication...
+Rank [4/16]: Starting iteration 1
+Rank [12/16]: Start tile-wise ring communication...
+Rank [12/16]: Starting iteration 1
+Rank [14/16]: Start tile-wise ring communication...
+Rank [14/16]: Starting iteration 1
+Rank [15/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 2
+Rank [13/16]: Starting iteration 2
+Rank [12/16]: Starting iteration 2
+Rank [11/16]: Starting iteration 2
+Rank [10/16]: Starting iteration 2
+Rank [9/16]: Starting iteration 2
+Rank [8/16]: Starting iteration 2
+Rank [7/16]: Starting iteration 2
+Rank [6/16]: Starting iteration 2
+Rank [5/16]: Starting iteration 2
+Rank [4/16]: Starting iteration 2
+Rank [3/16]: Starting iteration 2
+Rank [2/16]: Starting iteration 2
+Rank [0/16]: Starting iteration 2
+Rank [1/16]: Starting iteration 2
+Rank [15/16]: Starting iteration 3
+Rank [14/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 3
+Rank [12/16]: Starting iteration 3
+Rank [11/16]: Starting iteration 3
+Rank [10/16]: Starting iteration 3
+Rank [8/16]: Starting iteration 3
+Rank [9/16]: Starting iteration 3
+Rank [7/16]: Starting iteration 3
+Rank [6/16]: Starting iteration 3
+Rank [5/16]: Starting iteration 3
+Rank [4/16]: Starting iteration 3
+Rank [1/16]: Starting iteration 3
+Rank [0/16]: Starting iteration 3
+Rank [2/16]: Starting iteration 3
+Rank [3/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 4
+Rank [14/16]: Starting iteration 4
+Rank [15/16]: Starting iteration 4
+Rank [10/16]: Starting iteration 4
+Rank [11/16]: Starting iteration 4
+Rank [12/16]: Starting iteration 4
+Rank [7/16]: Starting iteration 4
+Rank [8/16]: Starting iteration 4
+Rank [9/16]: Starting iteration 4
+Rank [2/16]: Starting iteration 4
+Rank [4/16]: Starting iteration 4
+Rank [1/16]: Starting iteration 4
+Rank [5/16]: Starting iteration 4
+Rank [6/16]: Starting iteration 4
+Rank [0/16]: Starting iteration 4
+Rank [3/16]: Starting iteration 4
+Rank [13/16]: Starting iteration 5
+Rank [14/16]: Starting iteration 5
+Rank [9/16]: Starting iteration 5
+Rank [12/16]: Starting iteration 5
+Rank [10/16]: Starting iteration 5
+Rank [15/16]: Starting iteration 5
+Rank [1/16]: Starting iteration 5
+Rank [5/16]: Starting iteration 5
+Rank [8/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 5
+Rank [2/16]: Starting iteration 5
+Rank [6/16]: Starting iteration 5
+Rank [0/16]: Starting iteration 5
+Rank [4/16]: Starting iteration 5
+Rank [7/16]: Starting iteration 5
+Rank [3/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 6
+Rank [15/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 6
+Rank [13/16]: Starting iteration 6
+Rank [1/16]: Starting iteration 6
+Rank [10/16]: Starting iteration 6
+Rank [2/16]: Starting iteration 6
+Rank [7/16]: Starting iteration 6
+Rank [6/16]: Starting iteration 6
+Rank [0/16]: Starting iteration 6
+Rank [5/16]: Starting iteration 6
+Rank [8/16]: Starting iteration 6
+Rank [14/16]: Starting iteration 6
+Rank [3/16]: Starting iteration 6
+Rank [4/16]: Starting iteration 6
+Rank [9/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 7
+Rank [13/16]: Starting iteration 7
+Rank [0/16]: Starting iteration 7
+Rank [5/16]: Starting iteration 7
+Rank [11/16]: Starting iteration 7
+Rank [6/16]: Starting iteration 7
+Rank [15/16]: Starting iteration 7
+Rank [4/16]: Starting iteration 7
+Rank [10/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 7
+Rank [7/16]: Starting iteration 7
+Rank [1/16]: Starting iteration 7
+Rank [3/16]: Starting iteration 7
+Rank [9/16]: Starting iteration 7
+Rank [2/16]: Starting iteration 7
+Rank [8/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 8
+Rank [15/16]: Starting iteration 8
+Rank [0/16]: Starting iteration 8
+Rank [7/16]: Starting iteration 8
+Rank [13/16]: Starting iteration 8
+Rank [6/16]: Starting iteration 8
+Rank [5/16]: Starting iteration 8
+Rank [12/16]: Starting iteration 8
+Rank [10/16]: Starting iteration 8
+Rank [1/16]: Starting iteration 8
+Rank [4/16]: Starting iteration 8
+Rank [2/16]: Starting iteration 8
+Rank [8/16]: Starting iteration 8
+Rank [11/16]: Starting iteration 8
+Rank [3/16]: Starting iteration 8
+Rank [9/16]: Starting iteration 8
+Rank [15/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.9820898378966376 s
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Run 3
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks...
+Rank [9/16]: Calculate diagonal blocks...
+Rank [8/16]: Calculate diagonal blocks...
+Rank [14/16]: Calculate diagonal blocks...
+Rank [3/16]: Calculate diagonal blocks...
+Rank [5/16]: Calculate diagonal blocks...
+Rank [2/16]: Calculate diagonal blocks...
+Rank [11/16]: Calculate diagonal blocks...
+Rank [12/16]: Calculate diagonal blocks...
+Rank [7/16]: Calculate diagonal blocks...
+Rank [13/16]: Calculate diagonal blocks...
+Rank [10/16]: Calculate diagonal blocks...
+Rank [1/16]: Calculate diagonal blocks...
+Rank [15/16]: Calculate diagonal blocks...
+Rank [6/16]: Calculate diagonal blocks...
+Rank [4/16]: Calculate diagonal blocks...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [0/16]: Starting iteration 1
+Rank [7/16]: Start tile-wise ring communication...
+Rank [7/16]: Starting iteration 1
+Rank [2/16]: Start tile-wise ring communication...
+Rank [2/16]: Starting iteration 1
+Rank [1/16]: Start tile-wise ring communication...
+Rank [1/16]: Starting iteration 1
+Rank [14/16]: Start tile-wise ring communication...
+Rank [14/16]: Starting iteration 1
+Rank [5/16]: Start tile-wise ring communication...
+Rank [5/16]: Starting iteration 1
+Rank [13/16]: Start tile-wise ring communication...
+Rank [13/16]: Starting iteration 1
+Rank [9/16]: Start tile-wise ring communication...
+Rank [9/16]: Starting iteration 1
+Rank [8/16]: Start tile-wise ring communication...
+Rank [8/16]: Starting iteration 1
+Rank [4/16]: Start tile-wise ring communication...
+Rank [4/16]: Starting iteration 1
+Rank [15/16]: Start tile-wise ring communication...
+Rank [15/16]: Starting iteration 1
+Rank [12/16]: Start tile-wise ring communication...
+Rank [12/16]: Starting iteration 1
+Rank [10/16]: Start tile-wise ring communication...
+Rank [10/16]: Starting iteration 1
+Rank [3/16]: Start tile-wise ring communication...
+Rank [3/16]: Starting iteration 1
+Rank [6/16]: Start tile-wise ring communication...
+Rank [6/16]: Starting iteration 1
+Rank [11/16]: Start tile-wise ring communication...
+Rank [11/16]: Starting iteration 1
+Rank [15/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 2
+Rank [13/16]: Starting iteration 2
+Rank [12/16]: Starting iteration 2
+Rank [11/16]: Starting iteration 2
+Rank [10/16]: Starting iteration 2
+Rank [9/16]: Starting iteration 2
+Rank [8/16]: Starting iteration 2
+Rank [7/16]: Starting iteration 2
+Rank [6/16]: Starting iteration 2
+Rank [5/16]: Starting iteration 2
+Rank [4/16]: Starting iteration 2
+Rank [3/16]: Starting iteration 2
+Rank [2/16]: Starting iteration 2
+Rank [0/16]: Starting iteration 2
+Rank [1/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 3
+Rank [15/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 3
+Rank [12/16]: Starting iteration 3
+Rank [11/16]: Starting iteration 3
+Rank [10/16]: Starting iteration 3
+Rank [8/16]: Starting iteration 3
+Rank [9/16]: Starting iteration 3
+Rank [7/16]: Starting iteration 3
+Rank [6/16]: Starting iteration 3
+Rank [5/16]: Starting iteration 3
+Rank [4/16]: Starting iteration 3
+Rank [1/16]: Starting iteration 3
+Rank [0/16]: Starting iteration 3
+Rank [2/16]: Starting iteration 3
+Rank [3/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 4
+Rank [14/16]: Starting iteration 4
+Rank [15/16]: Starting iteration 4
+Rank [11/16]: Starting iteration 4
+Rank [10/16]: Starting iteration 4
+Rank [12/16]: Starting iteration 4
+Rank [7/16]: Starting iteration 4
+Rank [8/16]: Starting iteration 4
+Rank [9/16]: Starting iteration 4
+Rank [1/16]: Starting iteration 4
+Rank [4/16]: Starting iteration 4
+Rank [2/16]: Starting iteration 4
+Rank [5/16]: Starting iteration 4
+Rank [6/16]: Starting iteration 4
+Rank [0/16]: Starting iteration 4
+Rank [3/16]: Starting iteration 4
+Rank [13/16]: Starting iteration 5
+Rank [14/16]: Starting iteration 5
+Rank [10/16]: Starting iteration 5
+Rank [12/16]: Starting iteration 5
+Rank [9/16]: Starting iteration 5
+Rank [15/16]: Starting iteration 5
+Rank [8/16]: Starting iteration 5
+Rank [1/16]: Starting iteration 5
+Rank [2/16]: Starting iteration 5
+Rank [5/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 5
+Rank [6/16]: Starting iteration 5
+Rank [0/16]: Starting iteration 5
+Rank [4/16]: Starting iteration 5
+Rank [7/16]: Starting iteration 5
+Rank [3/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 6
+Rank [15/16]: Starting iteration 6
+Rank [13/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 6
+Rank [1/16]: Starting iteration 6
+Rank [10/16]: Starting iteration 6
+Rank [2/16]: Starting iteration 6
+Rank [6/16]: Starting iteration 6
+Rank [7/16]: Starting iteration 6
+Rank [0/16]: Starting iteration 6
+Rank [5/16]: Starting iteration 6
+Rank [14/16]: Starting iteration 6
+Rank [3/16]: Starting iteration 6
+Rank [8/16]: Starting iteration 6
+Rank [4/16]: Starting iteration 6
+Rank [9/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 7
+Rank [0/16]: Starting iteration 7
+Rank [13/16]: Starting iteration 7
+Rank [5/16]: Starting iteration 7
+Rank [4/16]: Starting iteration 7
+Rank [10/16]: Starting iteration 7
+Rank [11/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 7
+Rank [6/16]: Starting iteration 7
+Rank [15/16]: Starting iteration 7
+Rank [1/16]: Starting iteration 7
+Rank [7/16]: Starting iteration 7
+Rank [2/16]: Starting iteration 7
+Rank [8/16]: Starting iteration 7
+Rank [3/16]: Starting iteration 7
+Rank [9/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 8
+Rank [13/16]: Starting iteration 8
+Rank [6/16]: Starting iteration 8
+Rank [15/16]: Starting iteration 8
+Rank [0/16]: Starting iteration 8
+Rank [7/16]: Starting iteration 8
+Rank [1/16]: Starting iteration 8
+Rank [8/16]: Starting iteration 8
+Rank [4/16]: Starting iteration 8
+Rank [2/16]: Starting iteration 8
+Rank [11/16]: Starting iteration 8
+Rank [9/16]: Starting iteration 8
+Rank [5/16]: Starting iteration 8
+Rank [10/16]: Starting iteration 8
+Rank [3/16]: Starting iteration 8
+Rank [12/16]: Starting iteration 8
+Rank [14/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [15/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.9744950910098851 s
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Run 4
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks...
+Rank [1/16]: Calculate diagonal blocks...
+Rank [3/16]: Calculate diagonal blocks...
+Rank [11/16]: Calculate diagonal blocks...
+Rank [8/16]: Calculate diagonal blocks...
+Rank [7/16]: Calculate diagonal blocks...
+Rank [6/16]: Calculate diagonal blocks...
+Rank [13/16]: Calculate diagonal blocks...
+Rank [2/16]: Calculate diagonal blocks...
+Rank [14/16]: Calculate diagonal blocks...
+Rank [4/16]: Calculate diagonal blocks...
+Rank [5/16]: Calculate diagonal blocks...
+Rank [12/16]: Calculate diagonal blocks...
+Rank [9/16]: Calculate diagonal blocks...
+Rank [10/16]: Calculate diagonal blocks...
+Rank [15/16]: Calculate diagonal blocks...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [0/16]: Starting iteration 1
+Rank [8/16]: Start tile-wise ring communication...
+Rank [8/16]: Starting iteration 1
+Rank [3/16]: Start tile-wise ring communication...
+Rank [3/16]: Starting iteration 1
+Rank [1/16]: Start tile-wise ring communication...
+Rank [1/16]: Starting iteration 1
+Rank [5/16]: Start tile-wise ring communication...
+Rank [5/16]: Starting iteration 1
+Rank [14/16]: Start tile-wise ring communication...
+Rank [14/16]: Starting iteration 1
+Rank [13/16]: Start tile-wise ring communication...
+Rank [13/16]: Starting iteration 1
+Rank [2/16]: Start tile-wise ring communication...
+Rank [2/16]: Starting iteration 1
+Rank [9/16]: Start tile-wise ring communication...
+Rank [9/16]: Starting iteration 1
+Rank [15/16]: Start tile-wise ring communication...
+Rank [15/16]: Starting iteration 1
+Rank [7/16]: Start tile-wise ring communication...
+Rank [7/16]: Starting iteration 1
+Rank [12/16]: Start tile-wise ring communication...
+Rank [12/16]: Starting iteration 1
+Rank [6/16]: Start tile-wise ring communication...
+Rank [6/16]: Starting iteration 1
+Rank [11/16]: Start tile-wise ring communication...
+Rank [11/16]: Starting iteration 1
+Rank [10/16]: Start tile-wise ring communication...
+Rank [10/16]: Starting iteration 1
+Rank [4/16]: Start tile-wise ring communication...
+Rank [4/16]: Starting iteration 1
+Rank [15/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 2
+Rank [13/16]: Starting iteration 2
+Rank [12/16]: Starting iteration 2
+Rank [11/16]: Starting iteration 2
+Rank [10/16]: Starting iteration 2
+Rank [9/16]: Starting iteration 2
+Rank [8/16]: Starting iteration 2
+Rank [7/16]: Starting iteration 2
+Rank [6/16]: Starting iteration 2
+Rank [5/16]: Starting iteration 2
+Rank [4/16]: Starting iteration 2
+Rank [3/16]: Starting iteration 2
+Rank [2/16]: Starting iteration 2
+Rank [0/16]: Starting iteration 2
+Rank [1/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 3
+Rank [15/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 3
+Rank [12/16]: Starting iteration 3
+Rank [10/16]: Starting iteration 3
+Rank [11/16]: Starting iteration 3
+Rank [8/16]: Starting iteration 3
+Rank [9/16]: Starting iteration 3
+Rank [7/16]: Starting iteration 3
+Rank [6/16]: Starting iteration 3
+Rank [4/16]: Starting iteration 3
+Rank [5/16]: Starting iteration 3
+Rank [0/16]: Starting iteration 3
+Rank [1/16]: Starting iteration 3
+Rank [2/16]: Starting iteration 3
+Rank [3/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 4
+Rank [14/16]: Starting iteration 4
+Rank [15/16]: Starting iteration 4
+Rank [10/16]: Starting iteration 4
+Rank [11/16]: Starting iteration 4
+Rank [12/16]: Starting iteration 4
+Rank [7/16]: Starting iteration 4
+Rank [8/16]: Starting iteration 4
+Rank [9/16]: Starting iteration 4
+Rank [1/16]: Starting iteration 4
+Rank [2/16]: Starting iteration 4
+Rank [4/16]: Starting iteration 4
+Rank [5/16]: Starting iteration 4
+Rank [6/16]: Starting iteration 4
+Rank [0/16]: Starting iteration 4
+Rank [3/16]: Starting iteration 4
+Rank [13/16]: Starting iteration 5
+Rank [14/16]: Starting iteration 5
+Rank [9/16]: Starting iteration 5
+Rank [12/16]: Starting iteration 5
+Rank [10/16]: Starting iteration 5
+Rank [15/16]: Starting iteration 5
+Rank [1/16]: Starting iteration 5
+Rank [5/16]: Starting iteration 5
+Rank [8/16]: Starting iteration 5
+Rank [2/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 5
+Rank [6/16]: Starting iteration 5
+Rank [0/16]: Starting iteration 5
+Rank [4/16]: Starting iteration 5
+Rank [7/16]: Starting iteration 5
+Rank [3/16]: Starting iteration 5
+Rank [15/16]: Starting iteration 6
+Rank [11/16]: Starting iteration 6
+Rank [13/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 6
+Rank [1/16]: Starting iteration 6
+Rank [10/16]: Starting iteration 6
+Rank [6/16]: Starting iteration 6
+Rank [2/16]: Starting iteration 6
+Rank [7/16]: Starting iteration 6
+Rank [0/16]: Starting iteration 6
+Rank [8/16]: Starting iteration 6
+Rank [14/16]: Starting iteration 6
+Rank [5/16]: Starting iteration 6
+Rank [3/16]: Starting iteration 6
+Rank [4/16]: Starting iteration 6
+Rank [9/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 7
+Rank [0/16]: Starting iteration 7
+Rank [6/16]: Starting iteration 7
+Rank [5/16]: Starting iteration 7
+Rank [11/16]: Starting iteration 7
+Rank [15/16]: Starting iteration 7
+Rank [13/16]: Starting iteration 7
+Rank [3/16]: Starting iteration 7
+Rank [9/16]: Starting iteration 7
+Rank [1/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 7
+Rank [10/16]: Starting iteration 7
+Rank [7/16]: Starting iteration 7
+Rank [4/16]: Starting iteration 7
+Rank [2/16]: Starting iteration 7
+Rank [8/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 8
+Rank [5/16]: Starting iteration 8
+Rank [12/16]: Starting iteration 8
+Rank [0/16]: Starting iteration 8
+Rank [7/16]: Starting iteration 8
+Rank [15/16]: Starting iteration 8
+Rank [1/16]: Starting iteration 8
+Rank [8/16]: Starting iteration 8
+Rank [10/16]: Starting iteration 8
+Rank [3/16]: Starting iteration 8
+Rank [13/16]: Starting iteration 8
+Rank [6/16]: Starting iteration 8
+Rank [2/16]: Starting iteration 8
+Rank [4/16]: Starting iteration 8
+Rank [11/16]: Starting iteration 8
+Rank [9/16]: Starting iteration 8
+Rank [15/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.9837135465350002 s
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Run 5
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks...
+Rank [9/16]: Calculate diagonal blocks...
+Rank [8/16]: Calculate diagonal blocks...
+Rank [10/16]: Calculate diagonal blocks...
+Rank [13/16]: Calculate diagonal blocks...
+Rank [4/16]: Calculate diagonal blocks...
+Rank [15/16]: Calculate diagonal blocks...
+Rank [3/16]: Calculate diagonal blocks...
+Rank [14/16]: Calculate diagonal blocks...
+Rank [7/16]: Calculate diagonal blocks...
+Rank [2/16]: Calculate diagonal blocks...
+Rank [5/16]: Calculate diagonal blocks...
+Rank [12/16]: Calculate diagonal blocks...
+Rank [11/16]: Calculate diagonal blocks...
+Rank [1/16]: Calculate diagonal blocks...
+Rank [6/16]: Calculate diagonal blocks...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [0/16]: Starting iteration 1
+Rank [10/16]: Start tile-wise ring communication...
+Rank [10/16]: Starting iteration 1
+Rank [7/16]: Start tile-wise ring communication...
+Rank [7/16]: Starting iteration 1
+Rank [5/16]: Start tile-wise ring communication...
+Rank [5/16]: Starting iteration 1
+Rank [12/16]: Start tile-wise ring communication...
+Rank [12/16]: Starting iteration 1
+Rank [9/16]: Start tile-wise ring communication...
+Rank [9/16]: Starting iteration 1
+Rank [11/16]: Start tile-wise ring communication...
+Rank [11/16]: Starting iteration 1
+Rank [8/16]: Start tile-wise ring communication...
+Rank [8/16]: Starting iteration 1
+Rank [15/16]: Start tile-wise ring communication...
+Rank [15/16]: Starting iteration 1
+Rank [6/16]: Start tile-wise ring communication...
+Rank [6/16]: Starting iteration 1
+Rank [1/16]: Start tile-wise ring communication...
+Rank [1/16]: Starting iteration 1
+Rank [4/16]: Start tile-wise ring communication...
+Rank [4/16]: Starting iteration 1
+Rank [13/16]: Start tile-wise ring communication...
+Rank [13/16]: Starting iteration 1
+Rank [14/16]: Start tile-wise ring communication...
+Rank [14/16]: Starting iteration 1
+Rank [3/16]: Start tile-wise ring communication...
+Rank [3/16]: Starting iteration 1
+Rank [2/16]: Start tile-wise ring communication...
+Rank [2/16]: Starting iteration 1
+Rank [15/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 2
+Rank [13/16]: Starting iteration 2
+Rank [12/16]: Starting iteration 2
+Rank [11/16]: Starting iteration 2
+Rank [10/16]: Starting iteration 2
+Rank [9/16]: Starting iteration 2
+Rank [8/16]: Starting iteration 2
+Rank [7/16]: Starting iteration 2
+Rank [6/16]: Starting iteration 2
+Rank [5/16]: Starting iteration 2
+Rank [4/16]: Starting iteration 2
+Rank [3/16]: Starting iteration 2
+Rank [2/16]: Starting iteration 2
+Rank [0/16]: Starting iteration 2
+Rank [1/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 3
+Rank [15/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 3
+Rank [12/16]: Starting iteration 3
+Rank [10/16]: Starting iteration 3
+Rank [11/16]: Starting iteration 3
+Rank [8/16]: Starting iteration 3
+Rank [9/16]: Starting iteration 3
+Rank [7/16]: Starting iteration 3
+Rank [6/16]: Starting iteration 3
+Rank [4/16]: Starting iteration 3
+Rank [5/16]: Starting iteration 3
+Rank [0/16]: Starting iteration 3
+Rank [1/16]: Starting iteration 3
+Rank [2/16]: Starting iteration 3
+Rank [3/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 4
+Rank [14/16]: Starting iteration 4
+Rank [15/16]: Starting iteration 4
+Rank [10/16]: Starting iteration 4
+Rank [11/16]: Starting iteration 4
+Rank [12/16]: Starting iteration 4
+Rank [8/16]: Starting iteration 4
+Rank [7/16]: Starting iteration 4
+Rank [9/16]: Starting iteration 4
+Rank [2/16]: Starting iteration 4
+Rank [5/16]: Starting iteration 4
+Rank [4/16]: Starting iteration 4
+Rank [1/16]: Starting iteration 4
+Rank [6/16]: Starting iteration 4
+Rank [0/16]: Starting iteration 4
+Rank [3/16]: Starting iteration 4
+Rank [13/16]: Starting iteration 5
+Rank [14/16]: Starting iteration 5
+Rank [9/16]: Starting iteration 5
+Rank [10/16]: Starting iteration 5
+Rank [12/16]: Starting iteration 5
+Rank [15/16]: Starting iteration 5
+Rank [1/16]: Starting iteration 5
+Rank [5/16]: Starting iteration 5
+Rank [8/16]: Starting iteration 5
+Rank [2/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 5
+Rank [6/16]: Starting iteration 5
+Rank [0/16]: Starting iteration 5
+Rank [4/16]: Starting iteration 5
+Rank [7/16]: Starting iteration 5
+Rank [3/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 6
+Rank [15/16]: Starting iteration 6
+Rank [13/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 6
+Rank [10/16]: Starting iteration 6
+Rank [2/16]: Starting iteration 6
+Rank [7/16]: Starting iteration 6
+Rank [1/16]: Starting iteration 6
+Rank [6/16]: Starting iteration 6
+Rank [5/16]: Starting iteration 6
+Rank [0/16]: Starting iteration 6
+Rank [14/16]: Starting iteration 6
+Rank [8/16]: Starting iteration 6
+Rank [3/16]: Starting iteration 6
+Rank [4/16]: Starting iteration 6
+Rank [9/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 7
+Rank [13/16]: Starting iteration 7
+Rank [0/16]: Starting iteration 7
+Rank [5/16]: Starting iteration 7
+Rank [11/16]: Starting iteration 7
+Rank [6/16]: Starting iteration 7
+Rank [4/16]: Starting iteration 7
+Rank [15/16]: Starting iteration 7
+Rank [10/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 7
+Rank [1/16]: Starting iteration 7
+Rank [7/16]: Starting iteration 7
+Rank [3/16]: Starting iteration 7
+Rank [2/16]: Starting iteration 7
+Rank [9/16]: Starting iteration 7
+Rank [8/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 8
+Rank [15/16]: Starting iteration 8
+Rank [7/16]: Starting iteration 8
+Rank [0/16]: Starting iteration 8
+Rank [8/16]: Starting iteration 8
+Rank [1/16]: Starting iteration 8
+Rank [13/16]: Starting iteration 8
+Rank [5/16]: Starting iteration 8
+Rank [10/16]: Starting iteration 8
+Rank [6/16]: Starting iteration 8
+Rank [3/16]: Starting iteration 8
+Rank [12/16]: Starting iteration 8
+Rank [4/16]: Starting iteration 8
+Rank [11/16]: Starting iteration 8
+Rank [2/16]: Starting iteration 8
+Rank [9/16]: Starting iteration 8
+Rank [15/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.9822769179008901 s
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Run 6
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks...
+Rank [6/16]: Calculate diagonal blocks...
+Rank [1/16]: Calculate diagonal blocks...
+Rank [8/16]: Calculate diagonal blocks...
+Rank [14/16]: Calculate diagonal blocks...
+Rank [9/16]: Calculate diagonal blocks...
+Rank [12/16]: Calculate diagonal blocks...
+Rank [4/16]: Calculate diagonal blocks...
+Rank [15/16]: Calculate diagonal blocks...
+Rank [10/16]: Calculate diagonal blocks...
+Rank [7/16]: Calculate diagonal blocks...
+Rank [2/16]: Calculate diagonal blocks...
+Rank [3/16]: Calculate diagonal blocks...
+Rank [13/16]: Calculate diagonal blocks...
+Rank [11/16]: Calculate diagonal blocks...
+Rank [5/16]: Calculate diagonal blocks...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [0/16]: Starting iteration 1
+Rank [2/16]: Start tile-wise ring communication...
+Rank [2/16]: Starting iteration 1
+Rank [9/16]: Start tile-wise ring communication...
+Rank [9/16]: Starting iteration 1
+Rank [1/16]: Start tile-wise ring communication...
+Rank [1/16]: Starting iteration 1
+Rank [7/16]: Start tile-wise ring communication...
+Rank [7/16]: Starting iteration 1
+Rank [10/16]: Start tile-wise ring communication...
+Rank [10/16]: Starting iteration 1
+Rank [6/16]: Start tile-wise ring communication...
+Rank [6/16]: Starting iteration 1
+Rank [13/16]: Start tile-wise ring communication...
+Rank [13/16]: Starting iteration 1
+Rank [11/16]: Start tile-wise ring communication...
+Rank [11/16]: Starting iteration 1
+Rank [5/16]: Start tile-wise ring communication...
+Rank [5/16]: Starting iteration 1
+Rank [8/16]: Start tile-wise ring communication...
+Rank [8/16]: Starting iteration 1
+Rank [4/16]: Start tile-wise ring communication...
+Rank [4/16]: Starting iteration 1
+Rank [15/16]: Start tile-wise ring communication...
+Rank [15/16]: Starting iteration 1
+Rank [12/16]: Start tile-wise ring communication...
+Rank [12/16]: Starting iteration 1
+Rank [14/16]: Start tile-wise ring communication...
+Rank [14/16]: Starting iteration 1
+Rank [3/16]: Start tile-wise ring communication...
+Rank [3/16]: Starting iteration 1
+Rank [15/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 2
+Rank [13/16]: Starting iteration 2
+Rank [12/16]: Starting iteration 2
+Rank [11/16]: Starting iteration 2
+Rank [10/16]: Starting iteration 2
+Rank [9/16]: Starting iteration 2
+Rank [8/16]: Starting iteration 2
+Rank [7/16]: Starting iteration 2
+Rank [6/16]: Starting iteration 2
+Rank [5/16]: Starting iteration 2
+Rank [4/16]: Starting iteration 2
+Rank [3/16]: Starting iteration 2
+Rank [2/16]: Starting iteration 2
+Rank [0/16]: Starting iteration 2
+Rank [1/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 3
+Rank [15/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 3
+Rank [12/16]: Starting iteration 3
+Rank [11/16]: Starting iteration 3
+Rank [10/16]: Starting iteration 3
+Rank [8/16]: Starting iteration 3
+Rank [9/16]: Starting iteration 3
+Rank [7/16]: Starting iteration 3
+Rank [6/16]: Starting iteration 3
+Rank [5/16]: Starting iteration 3
+Rank [4/16]: Starting iteration 3
+Rank [1/16]: Starting iteration 3
+Rank [0/16]: Starting iteration 3
+Rank [2/16]: Starting iteration 3
+Rank [3/16]: Starting iteration 3
+Rank [14/16]: Starting iteration 4
+Rank [13/16]: Starting iteration 4
+Rank [15/16]: Starting iteration 4
+Rank [10/16]: Starting iteration 4
+Rank [11/16]: Starting iteration 4
+Rank [12/16]: Starting iteration 4
+Rank [7/16]: Starting iteration 4
+Rank [8/16]: Starting iteration 4
+Rank [9/16]: Starting iteration 4
+Rank [1/16]: Starting iteration 4
+Rank [4/16]: Starting iteration 4
+Rank [2/16]: Starting iteration 4
+Rank [5/16]: Starting iteration 4
+Rank [6/16]: Starting iteration 4
+Rank [0/16]: Starting iteration 4
+Rank [3/16]: Starting iteration 4
+Rank [14/16]: Starting iteration 5
+Rank [13/16]: Starting iteration 5
+Rank [10/16]: Starting iteration 5
+Rank [12/16]: Starting iteration 5
+Rank [9/16]: Starting iteration 5
+Rank [15/16]: Starting iteration 5
+Rank [8/16]: Starting iteration 5
+Rank [2/16]: Starting iteration 5
+Rank [1/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 5
+Rank [5/16]: Starting iteration 5
+Rank [6/16]: Starting iteration 5
+Rank [0/16]: Starting iteration 5
+Rank [4/16]: Starting iteration 5
+Rank [7/16]: Starting iteration 5
+Rank [3/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 6
+Rank [15/16]: Starting iteration 6
+Rank [13/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 6
+Rank [1/16]: Starting iteration 6
+Rank [10/16]: Starting iteration 6
+Rank [6/16]: Starting iteration 6
+Rank [2/16]: Starting iteration 6
+Rank [7/16]: Starting iteration 6
+Rank [0/16]: Starting iteration 6
+Rank [5/16]: Starting iteration 6
+Rank [14/16]: Starting iteration 6
+Rank [8/16]: Starting iteration 6
+Rank [3/16]: Starting iteration 6
+Rank [4/16]: Starting iteration 6
+Rank [9/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 7
+Rank [0/16]: Starting iteration 7
+Rank [13/16]: Starting iteration 7
+Rank [6/16]: Starting iteration 7
+Rank [11/16]: Starting iteration 7
+Rank [5/16]: Starting iteration 7
+Rank [4/16]: Starting iteration 7
+Rank [10/16]: Starting iteration 7
+Rank [15/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 7
+Rank [1/16]: Starting iteration 7
+Rank [7/16]: Starting iteration 7
+Rank [3/16]: Starting iteration 7
+Rank [9/16]: Starting iteration 7
+Rank [2/16]: Starting iteration 7
+Rank [8/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 8
+Rank [15/16]: Starting iteration 8
+Rank [0/16]: Starting iteration 8
+Rank [7/16]: Starting iteration 8
+Rank [2/16]: Starting iteration 8
+Rank [9/16]: Starting iteration 8
+Rank [11/16]: Starting iteration 8
+Rank [4/16]: Starting iteration 8
+Rank [13/16]: Starting iteration 8
+Rank [6/16]: Starting iteration 8
+Rank [5/16]: Starting iteration 8
+Rank [12/16]: Starting iteration 8
+Rank [1/16]: Starting iteration 8
+Rank [10/16]: Starting iteration 8
+Rank [8/16]: Starting iteration 8
+Rank [3/16]: Starting iteration 8
+Rank [15/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.9830903091933578 s
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Run 7
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks...
+Rank [9/16]: Calculate diagonal blocks...
+Rank [7/16]: Calculate diagonal blocks...
+Rank [15/16]: Calculate diagonal blocks...
+Rank [4/16]: Calculate diagonal blocks...
+Rank [14/16]: Calculate diagonal blocks...
+Rank [12/16]: Calculate diagonal blocks...
+Rank [8/16]: Calculate diagonal blocks...
+Rank [13/16]: Calculate diagonal blocks...
+Rank [1/16]: Calculate diagonal blocks...
+Rank [3/16]: Calculate diagonal blocks...
+Rank [11/16]: Calculate diagonal blocks...
+Rank [2/16]: Calculate diagonal blocks...
+Rank [5/16]: Calculate diagonal blocks...
+Rank [6/16]: Calculate diagonal blocks...
+Rank [10/16]: Calculate diagonal blocks...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [0/16]: Starting iteration 1
+Rank [5/16]: Start tile-wise ring communication...
+Rank [5/16]: Starting iteration 1
+Rank [9/16]: Start tile-wise ring communication...
+Rank [9/16]: Starting iteration 1
+Rank [1/16]: Start tile-wise ring communication...
+Rank [1/16]: Starting iteration 1
+Rank [3/16]: Start tile-wise ring communication...
+Rank [3/16]: Starting iteration 1
+Rank [7/16]: Start tile-wise ring communication...
+Rank [7/16]: Starting iteration 1
+Rank [15/16]: Start tile-wise ring communication...
+Rank [15/16]: Starting iteration 1
+Rank [4/16]: Start tile-wise ring communication...
+Rank [4/16]: Starting iteration 1
+Rank [11/16]: Start tile-wise ring communication...
+Rank [11/16]: Starting iteration 1
+Rank [6/16]: Start tile-wise ring communication...
+Rank [6/16]: Starting iteration 1
+Rank [8/16]: Start tile-wise ring communication...
+Rank [8/16]: Starting iteration 1
+Rank [13/16]: Start tile-wise ring communication...
+Rank [13/16]: Starting iteration 1
+Rank [2/16]: Start tile-wise ring communication...
+Rank [2/16]: Starting iteration 1
+Rank [10/16]: Start tile-wise ring communication...
+Rank [10/16]: Starting iteration 1
+Rank [12/16]: Start tile-wise ring communication...
+Rank [12/16]: Starting iteration 1
+Rank [14/16]: Start tile-wise ring communication...
+Rank [14/16]: Starting iteration 1
+Rank [15/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 2
+Rank [13/16]: Starting iteration 2
+Rank [12/16]: Starting iteration 2
+Rank [11/16]: Starting iteration 2
+Rank [10/16]: Starting iteration 2
+Rank [9/16]: Starting iteration 2
+Rank [8/16]: Starting iteration 2
+Rank [7/16]: Starting iteration 2
+Rank [6/16]: Starting iteration 2
+Rank [5/16]: Starting iteration 2
+Rank [4/16]: Starting iteration 2
+Rank [3/16]: Starting iteration 2
+Rank [2/16]: Starting iteration 2
+Rank [0/16]: Starting iteration 2
+Rank [1/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 3
+Rank [15/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 3
+Rank [12/16]: Starting iteration 3
+Rank [10/16]: Starting iteration 3
+Rank [11/16]: Starting iteration 3
+Rank [8/16]: Starting iteration 3
+Rank [9/16]: Starting iteration 3
+Rank [7/16]: Starting iteration 3
+Rank [6/16]: Starting iteration 3
+Rank [5/16]: Starting iteration 3
+Rank [4/16]: Starting iteration 3
+Rank [1/16]: Starting iteration 3
+Rank [0/16]: Starting iteration 3
+Rank [2/16]: Starting iteration 3
+Rank [3/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 4
+Rank [14/16]: Starting iteration 4
+Rank [15/16]: Starting iteration 4
+Rank [10/16]: Starting iteration 4
+Rank [11/16]: Starting iteration 4
+Rank [12/16]: Starting iteration 4
+Rank [7/16]: Starting iteration 4
+Rank [8/16]: Starting iteration 4
+Rank [9/16]: Starting iteration 4
+Rank [1/16]: Starting iteration 4
+Rank [4/16]: Starting iteration 4
+Rank [2/16]: Starting iteration 4
+Rank [5/16]: Starting iteration 4
+Rank [6/16]: Starting iteration 4
+Rank [0/16]: Starting iteration 4
+Rank [3/16]: Starting iteration 4
+Rank [13/16]: Starting iteration 5
+Rank [14/16]: Starting iteration 5
+Rank [9/16]: Starting iteration 5
+Rank [12/16]: Starting iteration 5
+Rank [10/16]: Starting iteration 5
+Rank [15/16]: Starting iteration 5
+Rank [1/16]: Starting iteration 5
+Rank [5/16]: Starting iteration 5
+Rank [8/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 5
+Rank [2/16]: Starting iteration 5
+Rank [6/16]: Starting iteration 5
+Rank [0/16]: Starting iteration 5
+Rank [4/16]: Starting iteration 5
+Rank [7/16]: Starting iteration 5
+Rank [3/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 6
+Rank [15/16]: Starting iteration 6
+Rank [13/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 6
+Rank [1/16]: Starting iteration 6
+Rank [6/16]: Starting iteration 6
+Rank [10/16]: Starting iteration 6
+Rank [2/16]: Starting iteration 6
+Rank [7/16]: Starting iteration 6
+Rank [0/16]: Starting iteration 6
+Rank [5/16]: Starting iteration 6
+Rank [14/16]: Starting iteration 6
+Rank [8/16]: Starting iteration 6
+Rank [3/16]: Starting iteration 6
+Rank [4/16]: Starting iteration 6
+Rank [9/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 7
+Rank [13/16]: Starting iteration 7
+Rank [0/16]: Starting iteration 7
+Rank [6/16]: Starting iteration 7
+Rank [5/16]: Starting iteration 7
+Rank [11/16]: Starting iteration 7
+Rank [15/16]: Starting iteration 7
+Rank [1/16]: Starting iteration 7
+Rank [4/16]: Starting iteration 7
+Rank [7/16]: Starting iteration 7
+Rank [10/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 7
+Rank [3/16]: Starting iteration 7
+Rank [9/16]: Starting iteration 7
+Rank [2/16]: Starting iteration 7
+Rank [8/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 8
+Rank [0/16]: Starting iteration 8
+Rank [7/16]: Starting iteration 8
+Rank [13/16]: Starting iteration 8
+Rank [6/16]: Starting iteration 8
+Rank [15/16]: Starting iteration 8
+Rank [5/16]: Starting iteration 8
+Rank [12/16]: Starting iteration 8
+Rank [1/16]: Starting iteration 8
+Rank [8/16]: Starting iteration 8
+Rank [10/16]: Starting iteration 8
+Rank [3/16]: Starting iteration 8
+Rank [2/16]: Starting iteration 8
+Rank [4/16]: Starting iteration 8
+Rank [11/16]: Starting iteration 8
+Rank [9/16]: Starting iteration 8
+Rank [15/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.9796913525788113 s
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Run 8
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks...
+Rank [3/16]: Calculate diagonal blocks...
+Rank [12/16]: Calculate diagonal blocks...
+Rank [1/16]: Calculate diagonal blocks...
+Rank [7/16]: Calculate diagonal blocks...
+Rank [11/16]: Calculate diagonal blocks...
+Rank [13/16]: Calculate diagonal blocks...
+Rank [15/16]: Calculate diagonal blocks...
+Rank [4/16]: Calculate diagonal blocks...
+Rank [2/16]: Calculate diagonal blocks...
+Rank [10/16]: Calculate diagonal blocks...
+Rank [9/16]: Calculate diagonal blocks...
+Rank [5/16]: Calculate diagonal blocks...
+Rank [6/16]: Calculate diagonal blocks...
+Rank [8/16]: Calculate diagonal blocks...
+Rank [14/16]: Calculate diagonal blocks...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [0/16]: Starting iteration 1
+Rank [8/16]: Start tile-wise ring communication...
+Rank [8/16]: Starting iteration 1
+Rank [13/16]: Start tile-wise ring communication...
+Rank [13/16]: Starting iteration 1
+Rank [15/16]: Start tile-wise ring communication...
+Rank [15/16]: Starting iteration 1
+Rank [2/16]: Start tile-wise ring communication...
+Rank [2/16]: Starting iteration 1
+Rank [10/16]: Start tile-wise ring communication...
+Rank [10/16]: Starting iteration 1
+Rank [9/16]: Start tile-wise ring communication...
+Rank [9/16]: Starting iteration 1
+Rank [5/16]: Start tile-wise ring communication...
+Rank [5/16]: Starting iteration 1
+Rank [3/16]: Start tile-wise ring communication...
+Rank [3/16]: Starting iteration 1
+Rank [6/16]: Start tile-wise ring communication...
+Rank [6/16]: Starting iteration 1
+Rank [7/16]: Start tile-wise ring communication...
+Rank [7/16]: Starting iteration 1
+Rank [4/16]: Start tile-wise ring communication...
+Rank [4/16]: Starting iteration 1
+Rank [12/16]: Start tile-wise ring communication...
+Rank [12/16]: Starting iteration 1
+Rank [1/16]: Start tile-wise ring communication...
+Rank [1/16]: Starting iteration 1
+Rank [14/16]: Start tile-wise ring communication...
+Rank [14/16]: Starting iteration 1
+Rank [11/16]: Start tile-wise ring communication...
+Rank [11/16]: Starting iteration 1
+Rank [15/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 2
+Rank [13/16]: Starting iteration 2
+Rank [12/16]: Starting iteration 2
+Rank [11/16]: Starting iteration 2
+Rank [10/16]: Starting iteration 2
+Rank [9/16]: Starting iteration 2
+Rank [8/16]: Starting iteration 2
+Rank [7/16]: Starting iteration 2
+Rank [6/16]: Starting iteration 2
+Rank [5/16]: Starting iteration 2
+Rank [4/16]: Starting iteration 2
+Rank [3/16]: Starting iteration 2
+Rank [2/16]: Starting iteration 2
+Rank [0/16]: Starting iteration 2
+Rank [1/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 3
+Rank [15/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 3
+Rank [12/16]: Starting iteration 3
+Rank [10/16]: Starting iteration 3
+Rank [11/16]: Starting iteration 3
+Rank [8/16]: Starting iteration 3
+Rank [9/16]: Starting iteration 3
+Rank [7/16]: Starting iteration 3
+Rank [6/16]: Starting iteration 3
+Rank [4/16]: Starting iteration 3
+Rank [5/16]: Starting iteration 3
+Rank [0/16]: Starting iteration 3
+Rank [2/16]: Starting iteration 3
+Rank [1/16]: Starting iteration 3
+Rank [3/16]: Starting iteration 3
+Rank [14/16]: Starting iteration 4
+Rank [13/16]: Starting iteration 4
+Rank [15/16]: Starting iteration 4
+Rank [11/16]: Starting iteration 4
+Rank [10/16]: Starting iteration 4
+Rank [12/16]: Starting iteration 4
+Rank [8/16]: Starting iteration 4
+Rank [7/16]: Starting iteration 4
+Rank [9/16]: Starting iteration 4
+Rank [2/16]: Starting iteration 4
+Rank [5/16]: Starting iteration 4
+Rank [1/16]: Starting iteration 4
+Rank [4/16]: Starting iteration 4
+Rank [6/16]: Starting iteration 4
+Rank [0/16]: Starting iteration 4
+Rank [3/16]: Starting iteration 4
+Rank [13/16]: Starting iteration 5
+Rank [14/16]: Starting iteration 5
+Rank [9/16]: Starting iteration 5
+Rank [10/16]: Starting iteration 5
+Rank [12/16]: Starting iteration 5
+Rank [15/16]: Starting iteration 5
+Rank [1/16]: Starting iteration 5
+Rank [5/16]: Starting iteration 5
+Rank [8/16]: Starting iteration 5
+Rank [2/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 5
+Rank [6/16]: Starting iteration 5
+Rank [0/16]: Starting iteration 5
+Rank [4/16]: Starting iteration 5
+Rank [7/16]: Starting iteration 5
+Rank [3/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 6
+Rank [15/16]: Starting iteration 6
+Rank [13/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 6
+Rank [1/16]: Starting iteration 6
+Rank [10/16]: Starting iteration 6
+Rank [14/16]: Starting iteration 6
+Rank [2/16]: Starting iteration 6
+Rank [6/16]: Starting iteration 6
+Rank [7/16]: Starting iteration 6
+Rank [8/16]: Starting iteration 6
+Rank [3/16]: Starting iteration 6
+Rank [0/16]: Starting iteration 6
+Rank [4/16]: Starting iteration 6
+Rank [5/16]: Starting iteration 6
+Rank [9/16]: Starting iteration 6
+Rank [5/16]: Starting iteration 7
+Rank [11/16]: Starting iteration 7
+Rank [12/16]: Starting iteration 7
+Rank [15/16]: Starting iteration 7
+Rank [0/16]: Starting iteration 7
+Rank [6/16]: Starting iteration 7
+Rank [13/16]: Starting iteration 7
+Rank [3/16]: Starting iteration 7
+Rank [9/16]: Starting iteration 7
+Rank [4/16]: Starting iteration 7
+Rank [10/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 7
+Rank [1/16]: Starting iteration 7
+Rank [7/16]: Starting iteration 7
+Rank [2/16]: Starting iteration 7
+Rank [8/16]: Starting iteration 7
+Rank [5/16]: Starting iteration 8
+Rank [14/16]: Starting iteration 8
+Rank [12/16]: Starting iteration 8
+Rank [15/16]: Starting iteration 8
+Rank [0/16]: Starting iteration 8
+Rank [7/16]: Starting iteration 8
+Rank [13/16]: Starting iteration 8
+Rank [6/16]: Starting iteration 8
+Rank [1/16]: Starting iteration 8
+Rank [10/16]: Starting iteration 8
+Rank [8/16]: Starting iteration 8
+Rank [3/16]: Starting iteration 8
+Rank [4/16]: Starting iteration 8
+Rank [2/16]: Starting iteration 8
+Rank [11/16]: Starting iteration 8
+Rank [9/16]: Starting iteration 8
+Rank [15/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.9835145988618024 s
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Run 9
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks...
+Rank [1/16]: Calculate diagonal blocks...
+Rank [3/16]: Calculate diagonal blocks...
+Rank [11/16]: Calculate diagonal blocks...
+Rank [12/16]: Calculate diagonal blocks...
+Rank [5/16]: Calculate diagonal blocks...
+Rank [2/16]: Calculate diagonal blocks...
+Rank [4/16]: Calculate diagonal blocks...
+Rank [7/16]: Calculate diagonal blocks...
+Rank [13/16]: Calculate diagonal blocks...
+Rank [14/16]: Calculate diagonal blocks...
+Rank [15/16]: Calculate diagonal blocks...
+Rank [8/16]: Calculate diagonal blocks...
+Rank [10/16]: Calculate diagonal blocks...
+Rank [6/16]: Calculate diagonal blocks...
+Rank [9/16]: Calculate diagonal blocks...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [0/16]: Starting iteration 1
+Rank [5/16]: Start tile-wise ring communication...
+Rank [5/16]: Starting iteration 1
+Rank [2/16]: Start tile-wise ring communication...
+Rank [2/16]: Starting iteration 1
+Rank [9/16]: Start tile-wise ring communication...
+Rank [9/16]: Starting iteration 1
+Rank [11/16]: Start tile-wise ring communication...
+Rank [11/16]: Starting iteration 1
+Rank [1/16]: Start tile-wise ring communication...
+Rank [1/16]: Starting iteration 1
+Rank [7/16]: Start tile-wise ring communication...
+Rank [7/16]: Starting iteration 1
+Rank [15/16]: Start tile-wise ring communication...
+Rank [15/16]: Starting iteration 1
+Rank [8/16]: Start tile-wise ring communication...
+Rank [8/16]: Starting iteration 1
+Rank [10/16]: Start tile-wise ring communication...
+Rank [10/16]: Starting iteration 1
+Rank [6/16]: Start tile-wise ring communication...
+Rank [6/16]: Starting iteration 1
+Rank [4/16]: Start tile-wise ring communication...
+Rank [4/16]: Starting iteration 1
+Rank [12/16]: Start tile-wise ring communication...
+Rank [12/16]: Starting iteration 1
+Rank [13/16]: Start tile-wise ring communication...
+Rank [13/16]: Starting iteration 1
+Rank [3/16]: Start tile-wise ring communication...
+Rank [3/16]: Starting iteration 1
+Rank [14/16]: Start tile-wise ring communication...
+Rank [14/16]: Starting iteration 1
+Rank [15/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 2
+Rank [13/16]: Starting iteration 2
+Rank [12/16]: Starting iteration 2
+Rank [11/16]: Starting iteration 2
+Rank [10/16]: Starting iteration 2
+Rank [9/16]: Starting iteration 2
+Rank [8/16]: Starting iteration 2
+Rank [7/16]: Starting iteration 2
+Rank [6/16]: Starting iteration 2
+Rank [5/16]: Starting iteration 2
+Rank [4/16]: Starting iteration 2
+Rank [3/16]: Starting iteration 2
+Rank [2/16]: Starting iteration 2
+Rank [0/16]: Starting iteration 2
+Rank [1/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 3
+Rank [15/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 3
+Rank [12/16]: Starting iteration 3
+Rank [10/16]: Starting iteration 3
+Rank [11/16]: Starting iteration 3
+Rank [8/16]: Starting iteration 3
+Rank [9/16]: Starting iteration 3
+Rank [7/16]: Starting iteration 3
+Rank [6/16]: Starting iteration 3
+Rank [4/16]: Starting iteration 3
+Rank [5/16]: Starting iteration 3
+Rank [0/16]: Starting iteration 3
+Rank [1/16]: Starting iteration 3
+Rank [2/16]: Starting iteration 3
+Rank [3/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 4
+Rank [14/16]: Starting iteration 4
+Rank [15/16]: Starting iteration 4
+Rank [11/16]: Starting iteration 4
+Rank [10/16]: Starting iteration 4
+Rank [12/16]: Starting iteration 4
+Rank [8/16]: Starting iteration 4
+Rank [7/16]: Starting iteration 4
+Rank [9/16]: Starting iteration 4
+Rank [2/16]: Starting iteration 4
+Rank [1/16]: Starting iteration 4
+Rank [4/16]: Starting iteration 4
+Rank [5/16]: Starting iteration 4
+Rank [6/16]: Starting iteration 4
+Rank [0/16]: Starting iteration 4
+Rank [3/16]: Starting iteration 4
+Rank [13/16]: Starting iteration 5
+Rank [14/16]: Starting iteration 5
+Rank [10/16]: Starting iteration 5
+Rank [12/16]: Starting iteration 5
+Rank [9/16]: Starting iteration 5
+Rank [15/16]: Starting iteration 5
+Rank [8/16]: Starting iteration 5
+Rank [1/16]: Starting iteration 5
+Rank [5/16]: Starting iteration 5
+Rank [2/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 5
+Rank [6/16]: Starting iteration 5
+Rank [0/16]: Starting iteration 5
+Rank [4/16]: Starting iteration 5
+Rank [7/16]: Starting iteration 5
+Rank [3/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 6
+Rank [15/16]: Starting iteration 6
+Rank [13/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 6
+Rank [1/16]: Starting iteration 6
+Rank [10/16]: Starting iteration 6
+Rank [6/16]: Starting iteration 6
+Rank [2/16]: Starting iteration 6
+Rank [7/16]: Starting iteration 6
+Rank [0/16]: Starting iteration 6
+Rank [5/16]: Starting iteration 6
+Rank [8/16]: Starting iteration 6
+Rank [14/16]: Starting iteration 6
+Rank [3/16]: Starting iteration 6
+Rank [4/16]: Starting iteration 6
+Rank [9/16]: Starting iteration 6
+Rank [5/16]: Starting iteration 7
+Rank [11/16]: Starting iteration 7
+Rank [15/16]: Starting iteration 7
+Rank [12/16]: Starting iteration 7
+Rank [0/16]: Starting iteration 7
+Rank [6/16]: Starting iteration 7
+Rank [13/16]: Starting iteration 7
+Rank [3/16]: Starting iteration 7
+Rank [9/16]: Starting iteration 7
+Rank [4/16]: Starting iteration 7
+Rank [10/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 7
+Rank [1/16]: Starting iteration 7
+Rank [7/16]: Starting iteration 7
+Rank [2/16]: Starting iteration 7
+Rank [8/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 8
+Rank [15/16]: Starting iteration 8
+Rank [0/16]: Starting iteration 8
+Rank [7/16]: Starting iteration 8
+Rank [5/16]: Starting iteration 8
+Rank [12/16]: Starting iteration 8
+Rank [13/16]: Starting iteration 8
+Rank [6/16]: Starting iteration 8
+Rank [1/16]: Starting iteration 8
+Rank [8/16]: Starting iteration 8
+Rank [10/16]: Starting iteration 8
+Rank [3/16]: Starting iteration 8
+Rank [2/16]: Starting iteration 8
+Rank [4/16]: Starting iteration 8
+Rank [11/16]: Starting iteration 8
+Rank [9/16]: Starting iteration 8
+Rank [15/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.9851604113355279 s
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Run 10
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks...
+Rank [7/16]: Calculate diagonal blocks...
+Rank [13/16]: Calculate diagonal blocks...
+Rank [6/16]: Calculate diagonal blocks...
+Rank [3/16]: Calculate diagonal blocks...
+Rank [12/16]: Calculate diagonal blocks...
+Rank [4/16]: Calculate diagonal blocks...
+Rank [14/16]: Calculate diagonal blocks...
+Rank [2/16]: Calculate diagonal blocks...
+Rank [8/16]: Calculate diagonal blocks...
+Rank [10/16]: Calculate diagonal blocks...
+Rank [1/16]: Calculate diagonal blocks...
+Rank [15/16]: Calculate diagonal blocks...
+Rank [9/16]: Calculate diagonal blocks...
+Rank [11/16]: Calculate diagonal blocks...
+Rank [5/16]: Calculate diagonal blocks...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [0/16]: Starting iteration 1
+Rank [4/16]: Start tile-wise ring communication...
+Rank [4/16]: Starting iteration 1
+Rank [9/16]: Start tile-wise ring communication...
+Rank [9/16]: Starting iteration 1
+Rank [8/16]: Start tile-wise ring communication...
+Rank [8/16]: Starting iteration 1
+Rank [13/16]: Start tile-wise ring communication...
+Rank [13/16]: Starting iteration 1
+Rank [10/16]: Start tile-wise ring communication...
+Rank [10/16]: Starting iteration 1
+Rank [3/16]: Start tile-wise ring communication...
+Rank [3/16]: Starting iteration 1
+Rank [7/16]: Start tile-wise ring communication...
+Rank [7/16]: Starting iteration 1
+Rank [15/16]: Start tile-wise ring communication...
+Rank [15/16]: Starting iteration 1
+Rank [11/16]: Start tile-wise ring communication...
+Rank [11/16]: Starting iteration 1
+Rank [1/16]: Start tile-wise ring communication...
+Rank [1/16]: Starting iteration 1
+Rank [5/16]: Start tile-wise ring communication...
+Rank [5/16]: Starting iteration 1
+Rank [6/16]: Start tile-wise ring communication...
+Rank [6/16]: Starting iteration 1
+Rank [12/16]: Start tile-wise ring communication...
+Rank [12/16]: Starting iteration 1
+Rank [14/16]: Start tile-wise ring communication...
+Rank [14/16]: Starting iteration 1
+Rank [2/16]: Start tile-wise ring communication...
+Rank [2/16]: Starting iteration 1
+Rank [15/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 2
+Rank [13/16]: Starting iteration 2
+Rank [12/16]: Starting iteration 2
+Rank [11/16]: Starting iteration 2
+Rank [10/16]: Starting iteration 2
+Rank [9/16]: Starting iteration 2
+Rank [8/16]: Starting iteration 2
+Rank [7/16]: Starting iteration 2
+Rank [6/16]: Starting iteration 2
+Rank [5/16]: Starting iteration 2
+Rank [4/16]: Starting iteration 2
+Rank [3/16]: Starting iteration 2
+Rank [2/16]: Starting iteration 2
+Rank [0/16]: Starting iteration 2
+Rank [1/16]: Starting iteration 2
+Rank [15/16]: Starting iteration 3
+Rank [14/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 3
+Rank [12/16]: Starting iteration 3
+Rank [11/16]: Starting iteration 3
+Rank [10/16]: Starting iteration 3
+Rank [8/16]: Starting iteration 3
+Rank [9/16]: Starting iteration 3
+Rank [7/16]: Starting iteration 3
+Rank [6/16]: Starting iteration 3
+Rank [5/16]: Starting iteration 3
+Rank [4/16]: Starting iteration 3
+Rank [0/16]: Starting iteration 3
+Rank [1/16]: Starting iteration 3
+Rank [2/16]: Starting iteration 3
+Rank [3/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 4
+Rank [14/16]: Starting iteration 4
+Rank [15/16]: Starting iteration 4
+Rank [10/16]: Starting iteration 4
+Rank [11/16]: Starting iteration 4
+Rank [12/16]: Starting iteration 4
+Rank [7/16]: Starting iteration 4
+Rank [8/16]: Starting iteration 4
+Rank [9/16]: Starting iteration 4
+Rank [1/16]: Starting iteration 4
+Rank [2/16]: Starting iteration 4
+Rank [5/16]: Starting iteration 4
+Rank [4/16]: Starting iteration 4
+Rank [6/16]: Starting iteration 4
+Rank [0/16]: Starting iteration 4
+Rank [3/16]: Starting iteration 4
+Rank [13/16]: Starting iteration 5
+Rank [14/16]: Starting iteration 5
+Rank [9/16]: Starting iteration 5
+Rank [10/16]: Starting iteration 5
+Rank [12/16]: Starting iteration 5
+Rank [15/16]: Starting iteration 5
+Rank [5/16]: Starting iteration 5
+Rank [1/16]: Starting iteration 5
+Rank [8/16]: Starting iteration 5
+Rank [2/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 5
+Rank [6/16]: Starting iteration 5
+Rank [0/16]: Starting iteration 5
+Rank [4/16]: Starting iteration 5
+Rank [7/16]: Starting iteration 5
+Rank [3/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 6
+Rank [15/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 6
+Rank [13/16]: Starting iteration 6
+Rank [1/16]: Starting iteration 6
+Rank [10/16]: Starting iteration 6
+Rank [2/16]: Starting iteration 6
+Rank [6/16]: Starting iteration 6
+Rank [7/16]: Starting iteration 6
+Rank [0/16]: Starting iteration 6
+Rank [5/16]: Starting iteration 6
+Rank [14/16]: Starting iteration 6
+Rank [8/16]: Starting iteration 6
+Rank [3/16]: Starting iteration 6
+Rank [4/16]: Starting iteration 6
+Rank [9/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 7
+Rank [13/16]: Starting iteration 7
+Rank [0/16]: Starting iteration 7
+Rank [6/16]: Starting iteration 7
+Rank [11/16]: Starting iteration 7
+Rank [5/16]: Starting iteration 7
+Rank [15/16]: Starting iteration 7
+Rank [4/16]: Starting iteration 7
+Rank [1/16]: Starting iteration 7
+Rank [10/16]: Starting iteration 7
+Rank [7/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 7
+Rank [3/16]: Starting iteration 7
+Rank [9/16]: Starting iteration 7
+Rank [2/16]: Starting iteration 7
+Rank [8/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 8
+Rank [0/16]: Starting iteration 8
+Rank [7/16]: Starting iteration 8
+Rank [15/16]: Starting iteration 8
+Rank [10/16]: Starting iteration 8
+Rank [5/16]: Starting iteration 8
+Rank [1/16]: Starting iteration 8
+Rank [8/16]: Starting iteration 8
+Rank [3/16]: Starting iteration 8
+Rank [12/16]: Starting iteration 8
+Rank [2/16]: Starting iteration 8
+Rank [9/16]: Starting iteration 8
+Rank [4/16]: Starting iteration 8
+Rank [11/16]: Starting iteration 8
+Rank [13/16]: Starting iteration 8
+Rank [6/16]: Starting iteration 8
+Rank [15/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.9738696235581301 s
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Run 11
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 16.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+Overall number of samples considered: 50000
+n^2/p = 156250000.0
+ [OK]
+Rank [0/16]: Local data chunk has shape [3125, 18]...
+Start distance calculations...
+ [OK]
+Rank [9/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [15/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [14/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [7/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [11/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [3/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [2/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [5/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [6/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [12/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [13/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [1/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [10/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [8/16]: Local data chunk has shape [3125, 18]...
+ [OK]
+Rank [4/16]: Local data chunk has shape [3125, 18]...
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Overall number of samples is 50000.
+Rank [0/16]: Calculate diagonal blocks...
+Rank [15/16]: Calculate diagonal blocks...
+Rank [14/16]: Calculate diagonal blocks...
+Rank [7/16]: Calculate diagonal blocks...
+Rank [5/16]: Calculate diagonal blocks...
+Rank [1/16]: Calculate diagonal blocks...
+Rank [13/16]: Calculate diagonal blocks...
+Rank [12/16]: Calculate diagonal blocks...
+Rank [3/16]: Calculate diagonal blocks...
+Rank [11/16]: Calculate diagonal blocks...
+Rank [4/16]: Calculate diagonal blocks...
+Rank [10/16]: Calculate diagonal blocks...
+Rank [2/16]: Calculate diagonal blocks...
+Rank [8/16]: Calculate diagonal blocks...
+Rank [9/16]: Calculate diagonal blocks...
+Rank [6/16]: Calculate diagonal blocks...
+Rank [0/16]: Start tile-wise ring communication...
+Rank [0/16]: Starting iteration 1
+Rank [7/16]: Start tile-wise ring communication...
+Rank [7/16]: Starting iteration 1
+Rank [5/16]: Start tile-wise ring communication...
+Rank [5/16]: Starting iteration 1
+Rank [15/16]: Start tile-wise ring communication...
+Rank [15/16]: Starting iteration 1
+Rank [2/16]: Start tile-wise ring communication...
+Rank [2/16]: Starting iteration 1
+Rank [13/16]: Start tile-wise ring communication...
+Rank [13/16]: Starting iteration 1
+Rank [3/16]: Start tile-wise ring communication...
+Rank [3/16]: Starting iteration 1
+Rank [11/16]: Start tile-wise ring communication...
+Rank [11/16]: Starting iteration 1
+Rank [10/16]: Start tile-wise ring communication...
+Rank [10/16]: Starting iteration 1
+Rank [6/16]: Start tile-wise ring communication...
+Rank [6/16]: Starting iteration 1
+Rank [12/16]: Start tile-wise ring communication...
+Rank [12/16]: Starting iteration 1
+Rank [8/16]: Start tile-wise ring communication...
+Rank [8/16]: Starting iteration 1
+Rank [4/16]: Start tile-wise ring communication...
+Rank [4/16]: Starting iteration 1
+Rank [9/16]: Start tile-wise ring communication...
+Rank [9/16]: Starting iteration 1
+Rank [1/16]: Start tile-wise ring communication...
+Rank [1/16]: Starting iteration 1
+Rank [14/16]: Start tile-wise ring communication...
+Rank [14/16]: Starting iteration 1
+Rank [15/16]: Starting iteration 2
+Rank [14/16]: Starting iteration 2
+Rank [13/16]: Starting iteration 2
+Rank [12/16]: Starting iteration 2
+Rank [11/16]: Starting iteration 2
+Rank [10/16]: Starting iteration 2
+Rank [9/16]: Starting iteration 2
+Rank [8/16]: Starting iteration 2
+Rank [7/16]: Starting iteration 2
+Rank [6/16]: Starting iteration 2
+Rank [5/16]: Starting iteration 2
+Rank [4/16]: Starting iteration 2
+Rank [3/16]: Starting iteration 2
+Rank [2/16]: Starting iteration 2
+Rank [0/16]: Starting iteration 2
+Rank [1/16]: Starting iteration 2
+Rank [15/16]: Starting iteration 3
+Rank [14/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 3
+Rank [12/16]: Starting iteration 3
+Rank [11/16]: Starting iteration 3
+Rank [10/16]: Starting iteration 3
+Rank [9/16]: Starting iteration 3
+Rank [8/16]: Starting iteration 3
+Rank [7/16]: Starting iteration 3
+Rank [6/16]: Starting iteration 3
+Rank [5/16]: Starting iteration 3
+Rank [4/16]: Starting iteration 3
+Rank [1/16]: Starting iteration 3
+Rank [0/16]: Starting iteration 3
+Rank [2/16]: Starting iteration 3
+Rank [3/16]: Starting iteration 3
+Rank [13/16]: Starting iteration 4
+Rank [14/16]: Starting iteration 4
+Rank [15/16]: Starting iteration 4
+Rank [10/16]: Starting iteration 4
+Rank [11/16]: Starting iteration 4
+Rank [12/16]: Starting iteration 4
+Rank [7/16]: Starting iteration 4
+Rank [9/16]: Starting iteration 4
+Rank [8/16]: Starting iteration 4
+Rank [1/16]: Starting iteration 4
+Rank [4/16]: Starting iteration 4
+Rank [2/16]: Starting iteration 4
+Rank [5/16]: Starting iteration 4
+Rank [6/16]: Starting iteration 4
+Rank [0/16]: Starting iteration 4
+Rank [3/16]: Starting iteration 4
+Rank [13/16]: Starting iteration 5
+Rank [14/16]: Starting iteration 5
+Rank [9/16]: Starting iteration 5
+Rank [12/16]: Starting iteration 5
+Rank [10/16]: Starting iteration 5
+Rank [15/16]: Starting iteration 5
+Rank [1/16]: Starting iteration 5
+Rank [5/16]: Starting iteration 5
+Rank [8/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 5
+Rank [2/16]: Starting iteration 5
+Rank [6/16]: Starting iteration 5
+Rank [0/16]: Starting iteration 5
+Rank [4/16]: Starting iteration 5
+Rank [7/16]: Starting iteration 5
+Rank [3/16]: Starting iteration 5
+Rank [11/16]: Starting iteration 6
+Rank [13/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 6
+Rank [15/16]: Starting iteration 6
+Rank [1/16]: Starting iteration 6
+Rank [2/16]: Starting iteration 6
+Rank [10/16]: Starting iteration 6
+Rank [7/16]: Starting iteration 6
+Rank [6/16]: Starting iteration 6
+Rank [0/16]: Starting iteration 6
+Rank [5/16]: Starting iteration 6
+Rank [14/16]: Starting iteration 6
+Rank [8/16]: Starting iteration 6
+Rank [3/16]: Starting iteration 6
+Rank [4/16]: Starting iteration 6
+Rank [9/16]: Starting iteration 6
+Rank [12/16]: Starting iteration 7
+Rank [13/16]: Starting iteration 7
+Rank [0/16]: Starting iteration 7
+Rank [6/16]: Starting iteration 7
+Rank [5/16]: Starting iteration 7
+Rank [11/16]: Starting iteration 7
+Rank [15/16]: Starting iteration 7
+Rank [4/16]: Starting iteration 7
+Rank [10/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 7
+Rank [1/16]: Starting iteration 7
+Rank [7/16]: Starting iteration 7
+Rank [3/16]: Starting iteration 7
+Rank [9/16]: Starting iteration 7
+Rank [2/16]: Starting iteration 7
+Rank [8/16]: Starting iteration 7
+Rank [14/16]: Starting iteration 8
+Rank [0/16]: Starting iteration 8
+Rank [7/16]: Starting iteration 8
+Rank [15/16]: Starting iteration 8
+Rank [13/16]: Starting iteration 8
+Rank [6/16]: Starting iteration 8
+Rank [4/16]: Starting iteration 8
+Rank [2/16]: Starting iteration 8
+Rank [11/16]: Starting iteration 8
+Rank [9/16]: Starting iteration 8
+Rank [1/16]: Starting iteration 8
+Rank [8/16]: Starting iteration 8
+Rank [5/16]: Starting iteration 8
+Rank [10/16]: Starting iteration 8
+Rank [3/16]: Starting iteration 8
+Rank [12/16]: Starting iteration 8
+Rank [15/16]: [DONE]
+Rank [14/16]: [DONE]
+Rank [7/16]: [DONE]
+Rank [6/16]: [DONE]
+Rank [8/16]: [DONE]
+Rank [9/16]: [DONE]
+Rank [10/16]: [DONE]
+Rank [13/16]: [DONE]
+Rank [11/16]: [DONE]
+Rank [12/16]: [DONE]
+Rank [0/16]: [DONE]
+Rank [1/16]: [DONE]
+Rank [5/16]: [DONE]
+Rank [2/16]: [DONE]
+Rank [3/16]: [DONE]
+Rank [4/16]: [DONE]
+Rank [0/16]: Local distance matrix has shape [3125, 50000].
+Process-averaged run time: 0.978876719425898 s
+Rank [4/16]: Local distance matrix has shape [3125, 50000].
+Rank [8/16]: Local distance matrix has shape [3125, 50000].
+Rank [5/16]: Local distance matrix has shape [3125, 50000].
+Rank [13/16]: Local distance matrix has shape [3125, 50000].
+Rank [3/16]: Local distance matrix has shape [3125, 50000].
+Rank [15/16]: Local distance matrix has shape [3125, 50000].
+Rank [14/16]: Local distance matrix has shape [3125, 50000].
+Rank [10/16]: Local distance matrix has shape [3125, 50000].
+Rank [2/16]: Local distance matrix has shape [3125, 50000].
+Rank [9/16]: Local distance matrix has shape [3125, 50000].
+Rank [6/16]: Local distance matrix has shape [3125, 50000].
+Rank [7/16]: Local distance matrix has shape [3125, 50000].
+Rank [1/16]: Local distance matrix has shape [3125, 50000].
+Rank [12/16]: Local distance matrix has shape [3125, 50000].
+Rank [11/16]: Local distance matrix has shape [3125, 50000].
+
+============================= JOB FEEDBACK =============================
+
+NodeName=uc2n[131-132,152,154,186-190,204-205,208,215-217,220]
+Job ID: 24894099
+Cluster: uc2
+User/Group: ku4408/scc
+State: COMPLETED (exit code 0)
+Nodes: 16
+Cores per node: 80
+CPU Utilized: 00:09:33
+CPU Efficiency: 0.27% of 2-09:57:20 core-walltime
+Job Wall-clock time: 00:02:43
+Memory Utilized: 1.14 GB
+Memory Efficiency: 0.08% of 1.37 TB
diff --git a/2_psrs_cdist/solutions/A1/weak_scaling_symmetric/nodes_2.out b/2_psrs_cdist/solutions/A1/weak_scaling_symmetric/nodes_2.out
new file mode 100644
index 0000000..27d9667
--- /dev/null
+++ b/2_psrs_cdist/solutions/A1/weak_scaling_symmetric/nodes_2.out
@@ -0,0 +1,323 @@
+Run 1
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+ [OK]
+Rank [0/2]: Local data chunk has shape [8839, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/2]: Local data chunk has shape [8839, 18]...
+Overall number of samples is 17678.
+Overall number of samples is 17678.
+Rank [0/2]: Calculate diagonal blocks...
+Rank [1/2]: Calculate diagonal blocks...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [0/2]: Starting iteration 1
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: Starting iteration 1
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [8839, 17678].
+Process-averaged run time: 2.8743350356817245 s
+Rank [1/2]: Local distance matrix has shape [8839, 17678].
+Run 2
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+ [OK]
+Rank [0/2]: Local data chunk has shape [8839, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/2]: Local data chunk has shape [8839, 18]...
+Overall number of samples is 17678.
+Overall number of samples is 17678.
+Rank [0/2]: Calculate diagonal blocks...
+Rank [1/2]: Calculate diagonal blocks...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [0/2]: Starting iteration 1
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: Starting iteration 1
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [8839, 17678].
+Process-averaged run time: 0.7791277188807726 s
+Rank [1/2]: Local distance matrix has shape [8839, 17678].
+Run 3
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+ [OK]
+Rank [1/2]: Local data chunk has shape [8839, 18]...
+ [OK]
+Rank [0/2]: Local data chunk has shape [8839, 18]...
+Start distance calculations...
+Overall number of samples is 17678.
+Overall number of samples is 17678.
+Rank [0/2]: Calculate diagonal blocks...
+Rank [1/2]: Calculate diagonal blocks...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [0/2]: Starting iteration 1
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: Starting iteration 1
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [8839, 17678].
+Process-averaged run time: 0.785643681883812 s
+Rank [1/2]: Local distance matrix has shape [8839, 17678].
+Run 4
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+ [OK]
+Rank [0/2]: Local data chunk has shape [8839, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/2]: Local data chunk has shape [8839, 18]...
+Overall number of samples is 17678.
+Overall number of samples is 17678.
+Rank [0/2]: Calculate diagonal blocks...
+Rank [1/2]: Calculate diagonal blocks...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [0/2]: Starting iteration 1
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: Starting iteration 1
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [8839, 17678].
+Process-averaged run time: 0.7803574101999402 s
+Rank [1/2]: Local distance matrix has shape [8839, 17678].
+Run 5
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+ [OK]
+Rank [1/2]: Local data chunk has shape [8839, 18]...
+ [OK]
+Rank [0/2]: Local data chunk has shape [8839, 18]...
+Start distance calculations...
+Overall number of samples is 17678.
+Overall number of samples is 17678.
+Rank [0/2]: Calculate diagonal blocks...
+Rank [1/2]: Calculate diagonal blocks...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [0/2]: Starting iteration 1
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: Starting iteration 1
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [8839, 17678].
+Process-averaged run time: 0.7792617054656148 s
+Rank [1/2]: Local distance matrix has shape [8839, 17678].
+Run 6
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+ [OK]
+Rank [1/2]: Local data chunk has shape [8839, 18]...
+ [OK]
+Rank [0/2]: Local data chunk has shape [8839, 18]...
+Start distance calculations...
+Overall number of samples is 17678.
+Overall number of samples is 17678.
+Rank [0/2]: Calculate diagonal blocks...
+Rank [1/2]: Calculate diagonal blocks...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [0/2]: Starting iteration 1
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: Starting iteration 1
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [8839, 17678].
+Process-averaged run time: 0.7786894557066262 s
+Rank [1/2]: Local distance matrix has shape [8839, 17678].
+Run 7
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+ [OK]
+Rank [0/2]: Local data chunk has shape [8839, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/2]: Local data chunk has shape [8839, 18]...
+Overall number of samples is 17678.
+Overall number of samples is 17678.
+Rank [0/2]: Calculate diagonal blocks...
+Rank [1/2]: Calculate diagonal blocks...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [0/2]: Starting iteration 1
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: Starting iteration 1
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [8839, 17678].
+Process-averaged run time: 0.7799867535941303 s
+Rank [1/2]: Local distance matrix has shape [8839, 17678].
+Run 8
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+ [OK]
+Rank [0/2]: Local data chunk has shape [8839, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/2]: Local data chunk has shape [8839, 18]...
+Overall number of samples is 17678.
+Overall number of samples is 17678.
+Rank [0/2]: Calculate diagonal blocks...
+Rank [1/2]: Calculate diagonal blocks...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [0/2]: Starting iteration 1
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: Starting iteration 1
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [8839, 17678].
+Process-averaged run time: 0.7817893326282501 s
+Rank [1/2]: Local distance matrix has shape [8839, 17678].
+Run 9
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+ [OK]
+Rank [0/2]: Local data chunk has shape [8839, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/2]: Local data chunk has shape [8839, 18]...
+Overall number of samples is 17678.
+Overall number of samples is 17678.
+Rank [0/2]: Calculate diagonal blocks...
+Rank [1/2]: Calculate diagonal blocks...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [0/2]: Starting iteration 1
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: Starting iteration 1
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [8839, 17678].
+Process-averaged run time: 0.7786144870333374 s
+Rank [1/2]: Local distance matrix has shape [8839, 17678].
+Run 10
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+ [OK]
+Rank [0/2]: Local data chunk has shape [8839, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/2]: Local data chunk has shape [8839, 18]...
+Overall number of samples is 17678.
+Overall number of samples is 17678.
+Rank [0/2]: Calculate diagonal blocks...
+Rank [1/2]: Calculate diagonal blocks...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [0/2]: Starting iteration 1
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: Starting iteration 1
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [8839, 17678].
+Process-averaged run time: 0.7789138527587056 s
+Rank [1/2]: Local distance matrix has shape [8839, 17678].
+Run 11
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 2.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+Overall number of samples considered: 17678
+n^2/p = 156255842.0
+ [OK]
+Rank [1/2]: Local data chunk has shape [8839, 18]...
+ [OK]
+Rank [0/2]: Local data chunk has shape [8839, 18]...
+Start distance calculations...
+Overall number of samples is 17678.
+Overall number of samples is 17678.
+Rank [0/2]: Calculate diagonal blocks...
+Rank [1/2]: Calculate diagonal blocks...
+Rank [0/2]: Start tile-wise ring communication...
+Rank [0/2]: Starting iteration 1
+Rank [1/2]: Start tile-wise ring communication...
+Rank [1/2]: Starting iteration 1
+Rank [1/2]: [DONE]
+Rank [0/2]: [DONE]
+Rank [0/2]: Local distance matrix has shape [8839, 17678].
+Process-averaged run time: 0.7775403885170817 s
+Rank [1/2]: Local distance matrix has shape [8839, 17678].
+
+============================= JOB FEEDBACK =============================
+
+NodeName=uc2n[007-008]
+Job ID: 24894095
+Cluster: uc2
+User/Group: ku4408/scc
+State: COMPLETED (exit code 0)
+Nodes: 2
+Cores per node: 80
+CPU Utilized: 00:01:06
+CPU Efficiency: 0.40% of 04:32:00 core-walltime
+Job Wall-clock time: 00:01:42
+Memory Utilized: 1.05 GB
+Memory Efficiency: 0.60% of 175.78 GB
diff --git a/2_psrs_cdist/solutions/A1/weak_scaling_symmetric/nodes_4.out b/2_psrs_cdist/solutions/A1/weak_scaling_symmetric/nodes_4.out
new file mode 100644
index 0000000..a1b3486
--- /dev/null
+++ b/2_psrs_cdist/solutions/A1/weak_scaling_symmetric/nodes_4.out
@@ -0,0 +1,587 @@
+Run 1
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+ [OK]
+Rank [2/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [3/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [0/4]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/4]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Rank [1/4]: Calculate diagonal blocks...
+Rank [2/4]: Calculate diagonal blocks...
+Rank [3/4]: Calculate diagonal blocks...
+Rank [0/4]: Calculate diagonal blocks...
+Rank [2/4]: Start tile-wise ring communication...
+Rank [2/4]: Starting iteration 1
+Rank [0/4]: Start tile-wise ring communication...
+Rank [0/4]: Starting iteration 1
+Rank [1/4]: Start tile-wise ring communication...
+Rank [1/4]: Starting iteration 1
+Rank [3/4]: Start tile-wise ring communication...
+Rank [3/4]: Starting iteration 1
+Rank [3/4]: Starting iteration 2
+Rank [2/4]: Starting iteration 2
+Rank [0/4]: Starting iteration 2
+Rank [1/4]: Starting iteration 2
+Rank [2/4]: [DONE]
+Rank [3/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [6250, 25000].
+Process-averaged run time: 0.8694815246853977 s
+Rank [1/4]: Local distance matrix has shape [6250, 25000].
+Rank [3/4]: Local distance matrix has shape [6250, 25000].
+Rank [2/4]: Local distance matrix has shape [6250, 25000].
+Run 2
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+ [OK]
+Rank [1/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [0/4]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [2/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [3/4]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Rank [0/4]: Calculate diagonal blocks...
+Rank [2/4]: Calculate diagonal blocks...
+Rank [1/4]: Calculate diagonal blocks...
+Rank [3/4]: Calculate diagonal blocks...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [0/4]: Starting iteration 1
+Rank [3/4]: Start tile-wise ring communication...
+Rank [3/4]: Starting iteration 1
+Rank [1/4]: Start tile-wise ring communication...
+Rank [1/4]: Starting iteration 1
+Rank [2/4]: Start tile-wise ring communication...
+Rank [2/4]: Starting iteration 1
+Rank [3/4]: Starting iteration 2
+Rank [2/4]: Starting iteration 2
+Rank [0/4]: Starting iteration 2
+Rank [1/4]: Starting iteration 2
+Rank [2/4]: [DONE]
+Rank [3/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [6250, 25000].
+Process-averaged run time: 0.7505762153305113 s
+Rank [1/4]: Local distance matrix has shape [6250, 25000].
+Rank [2/4]: Local distance matrix has shape [6250, 25000].
+Rank [3/4]: Local distance matrix has shape [6250, 25000].
+Run 3
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+ [OK]
+Rank [1/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [3/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [0/4]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [2/4]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Rank [0/4]: Calculate diagonal blocks...
+Rank [2/4]: Calculate diagonal blocks...
+Rank [1/4]: Calculate diagonal blocks...
+Rank [3/4]: Calculate diagonal blocks...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [0/4]: Starting iteration 1
+Rank [3/4]: Start tile-wise ring communication...
+Rank [3/4]: Starting iteration 1
+Rank [2/4]: Start tile-wise ring communication...
+Rank [2/4]: Starting iteration 1
+Rank [1/4]: Start tile-wise ring communication...
+Rank [1/4]: Starting iteration 1
+Rank [3/4]: Starting iteration 2
+Rank [2/4]: Starting iteration 2
+Rank [0/4]: Starting iteration 2
+Rank [1/4]: Starting iteration 2
+Rank [2/4]: [DONE]
+Rank [3/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [6250, 25000].
+Process-averaged run time: 0.7515064526814967 s
+Rank [2/4]: Local distance matrix has shape [6250, 25000].
+Rank [1/4]: Local distance matrix has shape [6250, 25000].
+Rank [3/4]: Local distance matrix has shape [6250, 25000].
+Run 4
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+ [OK]
+Rank [1/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [0/4]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [3/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [2/4]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Rank [0/4]: Calculate diagonal blocks...
+Rank [1/4]: Calculate diagonal blocks...
+Rank [2/4]: Calculate diagonal blocks...
+Rank [3/4]: Calculate diagonal blocks...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [0/4]: Starting iteration 1
+Rank [1/4]: Start tile-wise ring communication...
+Rank [1/4]: Starting iteration 1
+Rank [2/4]: Start tile-wise ring communication...
+Rank [2/4]: Starting iteration 1
+Rank [3/4]: Start tile-wise ring communication...
+Rank [3/4]: Starting iteration 1
+Rank [3/4]: Starting iteration 2
+Rank [2/4]: Starting iteration 2
+Rank [1/4]: Starting iteration 2
+Rank [0/4]: Starting iteration 2
+Rank [2/4]: [DONE]
+Rank [3/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [6250, 25000].
+Process-averaged run time: 0.7528263693675399 s
+Rank [3/4]: Local distance matrix has shape [6250, 25000].
+Rank [1/4]: Local distance matrix has shape [6250, 25000].
+Rank [2/4]: Local distance matrix has shape [6250, 25000].
+Run 5
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+ [OK]
+Rank [0/4]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [2/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [3/4]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Rank [0/4]: Calculate diagonal blocks...
+Rank [1/4]: Calculate diagonal blocks...
+Rank [3/4]: Calculate diagonal blocks...
+Rank [2/4]: Calculate diagonal blocks...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [0/4]: Starting iteration 1
+Rank [3/4]: Start tile-wise ring communication...
+Rank [3/4]: Starting iteration 1
+Rank [1/4]: Start tile-wise ring communication...
+Rank [1/4]: Starting iteration 1
+Rank [2/4]: Start tile-wise ring communication...
+Rank [2/4]: Starting iteration 1
+Rank [3/4]: Starting iteration 2
+Rank [2/4]: Starting iteration 2
+Rank [0/4]: Starting iteration 2
+Rank [1/4]: Starting iteration 2
+Rank [2/4]: [DONE]
+Rank [3/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [6250, 25000].
+Process-averaged run time: 0.7489524255506694 s
+Rank [1/4]: Local distance matrix has shape [6250, 25000].
+Rank [3/4]: Local distance matrix has shape [6250, 25000].
+Rank [2/4]: Local distance matrix has shape [6250, 25000].
+Run 6
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+ [OK]
+Rank [0/4]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [2/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [3/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [1/4]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Rank [0/4]: Calculate diagonal blocks...
+Rank [2/4]: Calculate diagonal blocks...
+Rank [1/4]: Calculate diagonal blocks...
+Rank [3/4]: Calculate diagonal blocks...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [0/4]: Starting iteration 1
+Rank [1/4]: Start tile-wise ring communication...
+Rank [1/4]: Starting iteration 1
+Rank [2/4]: Start tile-wise ring communication...
+Rank [2/4]: Starting iteration 1
+Rank [3/4]: Start tile-wise ring communication...
+Rank [3/4]: Starting iteration 1
+Rank [3/4]: Starting iteration 2
+Rank [2/4]: Starting iteration 2
+Rank [0/4]: Starting iteration 2
+Rank [1/4]: Starting iteration 2
+Rank [2/4]: [DONE]
+Rank [3/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [6250, 25000].
+Process-averaged run time: 0.7573314954061061 s
+Rank [2/4]: Local distance matrix has shape [6250, 25000].
+Rank [1/4]: Local distance matrix has shape [6250, 25000].
+Rank [3/4]: Local distance matrix has shape [6250, 25000].
+Run 7
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+ [OK]
+Rank [2/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [0/4]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [1/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [3/4]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Rank [0/4]: Calculate diagonal blocks...
+Rank [1/4]: Calculate diagonal blocks...
+Rank [2/4]: Calculate diagonal blocks...
+Rank [3/4]: Calculate diagonal blocks...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [0/4]: Starting iteration 1
+Rank [3/4]: Start tile-wise ring communication...
+Rank [3/4]: Starting iteration 1
+Rank [2/4]: Start tile-wise ring communication...
+Rank [2/4]: Starting iteration 1
+Rank [1/4]: Start tile-wise ring communication...
+Rank [1/4]: Starting iteration 1
+Rank [3/4]: Starting iteration 2
+Rank [2/4]: Starting iteration 2
+Rank [0/4]: Starting iteration 2
+Rank [1/4]: Starting iteration 2
+Rank [2/4]: [DONE]
+Rank [3/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [6250, 25000].
+Process-averaged run time: 0.7562281165737659 s
+Rank [2/4]: Local distance matrix has shape [6250, 25000].
+Rank [1/4]: Local distance matrix has shape [6250, 25000].
+Rank [3/4]: Local distance matrix has shape [6250, 25000].
+Run 8
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+ [OK]
+Rank [0/4]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [3/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [2/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [1/4]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Rank [0/4]: Calculate diagonal blocks...
+Rank [1/4]: Calculate diagonal blocks...
+Rank [3/4]: Calculate diagonal blocks...
+Rank [2/4]: Calculate diagonal blocks...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [0/4]: Starting iteration 1
+Rank [2/4]: Start tile-wise ring communication...
+Rank [2/4]: Starting iteration 1
+Rank [1/4]: Start tile-wise ring communication...
+Rank [1/4]: Starting iteration 1
+Rank [3/4]: Start tile-wise ring communication...
+Rank [3/4]: Starting iteration 1
+Rank [3/4]: Starting iteration 2
+Rank [2/4]: Starting iteration 2
+Rank [0/4]: Starting iteration 2
+Rank [1/4]: Starting iteration 2
+Rank [2/4]: [DONE]
+Rank [3/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [6250, 25000].
+Process-averaged run time: 0.7500242050737143 s
+Rank [1/4]: Local distance matrix has shape [6250, 25000].
+Rank [3/4]: Local distance matrix has shape [6250, 25000].
+Rank [2/4]: Local distance matrix has shape [6250, 25000].
+Run 9
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+ [OK]
+Rank [1/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [3/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [0/4]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [2/4]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Rank [0/4]: Calculate diagonal blocks...
+Rank [2/4]: Calculate diagonal blocks...
+Rank [1/4]: Calculate diagonal blocks...
+Rank [3/4]: Calculate diagonal blocks...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [0/4]: Starting iteration 1
+Rank [2/4]: Start tile-wise ring communication...
+Rank [2/4]: Starting iteration 1
+Rank [1/4]: Start tile-wise ring communication...
+Rank [1/4]: Starting iteration 1
+Rank [3/4]: Start tile-wise ring communication...
+Rank [3/4]: Starting iteration 1
+Rank [3/4]: Starting iteration 2
+Rank [2/4]: Starting iteration 2
+Rank [0/4]: Starting iteration 2
+Rank [1/4]: Starting iteration 2
+Rank [2/4]: [DONE]
+Rank [3/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [6250, 25000].
+Process-averaged run time: 0.7509703047107905 s
+Rank [1/4]: Local distance matrix has shape [6250, 25000].
+Rank [3/4]: Local distance matrix has shape [6250, 25000].
+Rank [2/4]: Local distance matrix has shape [6250, 25000].
+Run 10
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+ [OK]
+Rank [0/4]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [3/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [1/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [2/4]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Rank [0/4]: Calculate diagonal blocks...
+Rank [1/4]: Calculate diagonal blocks...
+Rank [3/4]: Calculate diagonal blocks...
+Rank [2/4]: Calculate diagonal blocks...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [0/4]: Starting iteration 1
+Rank [3/4]: Start tile-wise ring communication...
+Rank [3/4]: Starting iteration 1
+Rank [1/4]: Start tile-wise ring communication...
+Rank [1/4]: Starting iteration 1
+Rank [2/4]: Start tile-wise ring communication...
+Rank [2/4]: Starting iteration 1
+Rank [3/4]: Starting iteration 2
+Rank [2/4]: Starting iteration 2
+Rank [0/4]: Starting iteration 2
+Rank [1/4]: Starting iteration 2
+Rank [2/4]: [DONE]
+Rank [3/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [6250, 25000].
+Process-averaged run time: 0.7518582695629448 s
+Rank [2/4]: Local distance matrix has shape [6250, 25000].
+Rank [1/4]: Local distance matrix has shape [6250, 25000].
+Rank [3/4]: Local distance matrix has shape [6250, 25000].
+Run 11
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 4.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+Overall number of samples considered: 25000
+n^2/p = 156250000.0
+ [OK]
+Rank [0/4]: Local data chunk has shape [6250, 18]...
+Start distance calculations...
+ [OK]
+Rank [3/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [1/4]: Local data chunk has shape [6250, 18]...
+ [OK]
+Rank [2/4]: Local data chunk has shape [6250, 18]...
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Overall number of samples is 25000.
+Rank [0/4]: Calculate diagonal blocks...
+Rank [2/4]: Calculate diagonal blocks...
+Rank [3/4]: Calculate diagonal blocks...
+Rank [1/4]: Calculate diagonal blocks...
+Rank [0/4]: Start tile-wise ring communication...
+Rank [0/4]: Starting iteration 1
+Rank [1/4]: Start tile-wise ring communication...
+Rank [1/4]: Starting iteration 1
+Rank [3/4]: Start tile-wise ring communication...
+Rank [3/4]: Starting iteration 1
+Rank [2/4]: Start tile-wise ring communication...
+Rank [2/4]: Starting iteration 1
+Rank [3/4]: Starting iteration 2
+Rank [2/4]: Starting iteration 2
+Rank [0/4]: Starting iteration 2
+Rank [1/4]: Starting iteration 2
+Rank [2/4]: [DONE]
+Rank [3/4]: [DONE]
+Rank [0/4]: [DONE]
+Rank [1/4]: [DONE]
+Rank [0/4]: Local distance matrix has shape [6250, 25000].
+Process-averaged run time: 0.7512615812011063 s
+Rank [1/4]: Local distance matrix has shape [6250, 25000].
+Rank [2/4]: Local distance matrix has shape [6250, 25000].
+Rank [3/4]: Local distance matrix has shape [6250, 25000].
+
+============================= JOB FEEDBACK =============================
+
+NodeName=uc2n[001-004]
+Job ID: 24894097
+Cluster: uc2
+User/Group: ku4408/scc
+State: COMPLETED (exit code 0)
+Nodes: 4
+Cores per node: 80
+CPU Utilized: 00:02:09
+CPU Efficiency: 0.41% of 08:42:40 core-walltime
+Job Wall-clock time: 00:01:38
+Memory Utilized: 485.30 MB
+Memory Efficiency: 0.13% of 351.56 GB
diff --git a/2_psrs_cdist/solutions/A1/weak_scaling_symmetric/nodes_8.out b/2_psrs_cdist/solutions/A1/weak_scaling_symmetric/nodes_8.out
new file mode 100644
index 0000000..dab1874
--- /dev/null
+++ b/2_psrs_cdist/solutions/A1/weak_scaling_symmetric/nodes_8.out
@@ -0,0 +1,1247 @@
+Run 1
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+ [OK]
+Rank [3/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [0/8]: Local data chunk has shape [4419, 18]...
+Start distance calculations...
+ [OK]
+Rank [2/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [5/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [4/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [4422, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [4419, 18]...
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Rank [2/8]: Calculate diagonal blocks...
+Rank [7/8]: Calculate diagonal blocks...
+Rank [4/8]: Calculate diagonal blocks...
+Rank [3/8]: Calculate diagonal blocks...
+Rank [6/8]: Calculate diagonal blocks...
+Rank [0/8]: Calculate diagonal blocks...
+Rank [1/8]: Calculate diagonal blocks...
+Rank [5/8]: Calculate diagonal blocks...
+Rank [3/8]: Start tile-wise ring communication...
+Rank [3/8]: Starting iteration 1
+Rank [6/8]: Start tile-wise ring communication...
+Rank [6/8]: Starting iteration 1
+Rank [1/8]: Start tile-wise ring communication...
+Rank [1/8]: Starting iteration 1
+Rank [2/8]: Start tile-wise ring communication...
+Rank [2/8]: Starting iteration 1
+Rank [4/8]: Start tile-wise ring communication...
+Rank [4/8]: Starting iteration 1
+Rank [5/8]: Start tile-wise ring communication...
+Rank [5/8]: Starting iteration 1
+Rank [7/8]: Start tile-wise ring communication...
+Rank [7/8]: Starting iteration 1
+Rank [0/8]: Start tile-wise ring communication...
+Rank [0/8]: Starting iteration 1
+Rank [7/8]: Starting iteration 2
+Rank [4/8]: Starting iteration 2
+Rank [3/8]: Starting iteration 2
+Rank [6/8]: Starting iteration 2
+Rank [0/8]: Starting iteration 2
+Rank [2/8]: Starting iteration 2
+Rank [5/8]: Starting iteration 2
+Rank [1/8]: Starting iteration 2
+Rank [7/8]: Starting iteration 3
+Rank [6/8]: Starting iteration 3
+Rank [4/8]: Starting iteration 3
+Rank [5/8]: Starting iteration 3
+Rank [0/8]: Starting iteration 3
+Rank [2/8]: Starting iteration 3
+Rank [3/8]: Starting iteration 3
+Rank [1/8]: Starting iteration 3
+Rank [7/8]: Starting iteration 4
+Rank [2/8]: Starting iteration 4
+Rank [5/8]: Starting iteration 4
+Rank [4/8]: Starting iteration 4
+Rank [6/8]: Starting iteration 4
+Rank [1/8]: Starting iteration 4
+Rank [0/8]: Starting iteration 4
+Rank [3/8]: Starting iteration 4
+Rank [6/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [7/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [4419, 35355].
+Process-averaged run time: 2.252527244621888 s
+Rank [5/8]: Local distance matrix has shape [4419, 35355].
+Rank [3/8]: Local distance matrix has shape [4419, 35355].
+Rank [2/8]: Local distance matrix has shape [4419, 35355].
+Rank [7/8]: Local distance matrix has shape [4422, 35355].
+Rank [4/8]: Local distance matrix has shape [4419, 35355].
+Rank [6/8]: Local distance matrix has shape [4419, 35355].
+Rank [1/8]: Local distance matrix has shape [4419, 35355].
+Run 2
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+ [OK]
+Rank [4/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [0/8]: Local data chunk has shape [4419, 18]...
+Start distance calculations...
+ [OK]
+Rank [3/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [5/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [4422, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [4419, 18]...
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Rank [0/8]: Calculate diagonal blocks...
+Rank [2/8]: Calculate diagonal blocks...
+Rank [1/8]: Calculate diagonal blocks...
+Rank [5/8]: Calculate diagonal blocks...
+Rank [3/8]: Calculate diagonal blocks...
+Rank [6/8]: Calculate diagonal blocks...
+Rank [4/8]: Calculate diagonal blocks...
+Rank [7/8]: Calculate diagonal blocks...
+Rank [2/8]: Start tile-wise ring communication...
+Rank [2/8]: Starting iteration 1
+Rank [4/8]: Start tile-wise ring communication...
+Rank [4/8]: Starting iteration 1
+Rank [6/8]: Start tile-wise ring communication...
+Rank [6/8]: Starting iteration 1
+Rank [3/8]: Start tile-wise ring communication...
+Rank [3/8]: Starting iteration 1
+Rank [7/8]: Start tile-wise ring communication...
+Rank [7/8]: Starting iteration 1
+Rank [5/8]: Start tile-wise ring communication...
+Rank [5/8]: Starting iteration 1
+Rank [0/8]: Start tile-wise ring communication...
+Rank [0/8]: Starting iteration 1
+Rank [1/8]: Start tile-wise ring communication...
+Rank [1/8]: Starting iteration 1
+Rank [7/8]: Starting iteration 2
+Rank [6/8]: Starting iteration 2
+Rank [5/8]: Starting iteration 2
+Rank [4/8]: Starting iteration 2
+Rank [3/8]: Starting iteration 2
+Rank [2/8]: Starting iteration 2
+Rank [0/8]: Starting iteration 2
+Rank [1/8]: Starting iteration 2
+Rank [7/8]: Starting iteration 3
+Rank [6/8]: Starting iteration 3
+Rank [4/8]: Starting iteration 3
+Rank [5/8]: Starting iteration 3
+Rank [0/8]: Starting iteration 3
+Rank [2/8]: Starting iteration 3
+Rank [3/8]: Starting iteration 3
+Rank [1/8]: Starting iteration 3
+Rank [7/8]: Starting iteration 4
+Rank [2/8]: Starting iteration 4
+Rank [5/8]: Starting iteration 4
+Rank [6/8]: Starting iteration 4
+Rank [4/8]: Starting iteration 4
+Rank [1/8]: Starting iteration 4
+Rank [0/8]: Starting iteration 4
+Rank [3/8]: Starting iteration 4
+Rank [6/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [7/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [4419, 35355].
+Process-averaged run time: 0.9125691251829267 s
+Rank [3/8]: Local distance matrix has shape [4419, 35355].
+Rank [1/8]: Local distance matrix has shape [4419, 35355].
+Rank [6/8]: Local distance matrix has shape [4419, 35355].
+Rank [4/8]: Local distance matrix has shape [4419, 35355].
+Rank [7/8]: Local distance matrix has shape [4422, 35355].
+Rank [2/8]: Local distance matrix has shape [4419, 35355].
+Rank [5/8]: Local distance matrix has shape [4419, 35355].
+Run 3
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+ [OK]
+Rank [0/8]: Local data chunk has shape [4419, 18]...
+Start distance calculations...
+ [OK]
+Rank [4/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [3/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [4422, 18]...
+ [OK]
+Rank [5/8]: Local data chunk has shape [4419, 18]...
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Rank [0/8]: Calculate diagonal blocks...
+Rank [2/8]: Calculate diagonal blocks...
+Rank [6/8]: Calculate diagonal blocks...
+Rank [3/8]: Calculate diagonal blocks...
+Rank [1/8]: Calculate diagonal blocks...
+Rank [5/8]: Calculate diagonal blocks...
+Rank [7/8]: Calculate diagonal blocks...
+Rank [4/8]: Calculate diagonal blocks...
+Rank [6/8]: Start tile-wise ring communication...
+Rank [6/8]: Starting iteration 1
+Rank [2/8]: Start tile-wise ring communication...
+Rank [2/8]: Starting iteration 1
+Rank [4/8]: Start tile-wise ring communication...
+Rank [4/8]: Starting iteration 1
+Rank [3/8]: Start tile-wise ring communication...
+Rank [3/8]: Starting iteration 1
+Rank [0/8]: Start tile-wise ring communication...
+Rank [0/8]: Starting iteration 1
+Rank [7/8]: Start tile-wise ring communication...
+Rank [7/8]: Starting iteration 1
+Rank [1/8]: Start tile-wise ring communication...
+Rank [1/8]: Starting iteration 1
+Rank [5/8]: Start tile-wise ring communication...
+Rank [5/8]: Starting iteration 1
+Rank [7/8]: Starting iteration 2
+Rank [6/8]: Starting iteration 2
+Rank [5/8]: Starting iteration 2
+Rank [4/8]: Starting iteration 2
+Rank [3/8]: Starting iteration 2
+Rank [2/8]: Starting iteration 2
+Rank [0/8]: Starting iteration 2
+Rank [1/8]: Starting iteration 2
+Rank [7/8]: Starting iteration 3
+Rank [6/8]: Starting iteration 3
+Rank [4/8]: Starting iteration 3
+Rank [5/8]: Starting iteration 3
+Rank [0/8]: Starting iteration 3
+Rank [2/8]: Starting iteration 3
+Rank [3/8]: Starting iteration 3
+Rank [1/8]: Starting iteration 3
+Rank [7/8]: Starting iteration 4
+Rank [2/8]: Starting iteration 4
+Rank [5/8]: Starting iteration 4
+Rank [6/8]: Starting iteration 4
+Rank [4/8]: Starting iteration 4
+Rank [1/8]: Starting iteration 4
+Rank [0/8]: Starting iteration 4
+Rank [3/8]: Starting iteration 4
+Rank [6/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [7/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [4419, 35355].
+Process-averaged run time: 0.911831810255535 s
+Rank [3/8]: Local distance matrix has shape [4419, 35355].
+Rank [2/8]: Local distance matrix has shape [4419, 35355].
+Rank [6/8]: Local distance matrix has shape [4419, 35355].
+Rank [1/8]: Local distance matrix has shape [4419, 35355].
+Rank [7/8]: Local distance matrix has shape [4422, 35355].
+Rank [4/8]: Local distance matrix has shape [4419, 35355].
+Rank [5/8]: Local distance matrix has shape [4419, 35355].
+Run 4
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+ [OK]
+Rank [0/8]: Local data chunk has shape [4419, 18]...
+Start distance calculations...
+ [OK]
+Rank [7/8]: Local data chunk has shape [4422, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [4/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [3/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [5/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [4419, 18]...
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Rank [0/8]: Calculate diagonal blocks...
+Rank [2/8]: Calculate diagonal blocks...
+Rank [3/8]: Calculate diagonal blocks...
+Rank [4/8]: Calculate diagonal blocks...
+Rank [7/8]: Calculate diagonal blocks...
+Rank [6/8]: Calculate diagonal blocks...
+Rank [1/8]: Calculate diagonal blocks...
+Rank [5/8]: Calculate diagonal blocks...
+Rank [4/8]: Start tile-wise ring communication...
+Rank [4/8]: Starting iteration 1
+Rank [0/8]: Start tile-wise ring communication...
+Rank [0/8]: Starting iteration 1
+Rank [2/8]: Start tile-wise ring communication...
+Rank [2/8]: Starting iteration 1
+Rank [1/8]: Start tile-wise ring communication...
+Rank [1/8]: Starting iteration 1
+Rank [6/8]: Start tile-wise ring communication...
+Rank [6/8]: Starting iteration 1
+Rank [5/8]: Start tile-wise ring communication...
+Rank [5/8]: Starting iteration 1
+Rank [7/8]: Start tile-wise ring communication...
+Rank [7/8]: Starting iteration 1
+Rank [3/8]: Start tile-wise ring communication...
+Rank [3/8]: Starting iteration 1
+Rank [7/8]: Starting iteration 2
+Rank [6/8]: Starting iteration 2
+Rank [5/8]: Starting iteration 2
+Rank [4/8]: Starting iteration 2
+Rank [3/8]: Starting iteration 2
+Rank [2/8]: Starting iteration 2
+Rank [0/8]: Starting iteration 2
+Rank [1/8]: Starting iteration 2
+Rank [7/8]: Starting iteration 3
+Rank [6/8]: Starting iteration 3
+Rank [4/8]: Starting iteration 3
+Rank [5/8]: Starting iteration 3
+Rank [0/8]: Starting iteration 3
+Rank [2/8]: Starting iteration 3
+Rank [3/8]: Starting iteration 3
+Rank [1/8]: Starting iteration 3
+Rank [7/8]: Starting iteration 4
+Rank [2/8]: Starting iteration 4
+Rank [5/8]: Starting iteration 4
+Rank [6/8]: Starting iteration 4
+Rank [4/8]: Starting iteration 4
+Rank [1/8]: Starting iteration 4
+Rank [3/8]: Starting iteration 4
+Rank [0/8]: Starting iteration 4
+Rank [6/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [7/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [4419, 35355].
+Process-averaged run time: 0.9128659731941298 s
+Rank [3/8]: Local distance matrix has shape [4419, 35355].
+Rank [4/8]: Local distance matrix has shape [4419, 35355].
+Rank [2/8]: Local distance matrix has shape [4419, 35355].
+Rank [7/8]: Local distance matrix has shape [4422, 35355].
+Rank [6/8]: Local distance matrix has shape [4419, 35355].
+Rank [1/8]: Local distance matrix has shape [4419, 35355].
+Rank [5/8]: Local distance matrix has shape [4419, 35355].
+Run 5
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+ [OK]
+Rank [5/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [4/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [0/8]: Local data chunk has shape [4419, 18]...
+Start distance calculations...
+ [OK]
+Rank [3/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [4422, 18]...
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Rank [0/8]: Calculate diagonal blocks...
+Rank [3/8]: Calculate diagonal blocks...
+Rank [5/8]: Calculate diagonal blocks...
+Rank [2/8]: Calculate diagonal blocks...
+Rank [4/8]: Calculate diagonal blocks...
+Rank [7/8]: Calculate diagonal blocks...
+Rank [1/8]: Calculate diagonal blocks...
+Rank [6/8]: Calculate diagonal blocks...
+Rank [2/8]: Start tile-wise ring communication...
+Rank [2/8]: Starting iteration 1
+Rank [0/8]: Start tile-wise ring communication...
+Rank [0/8]: Starting iteration 1
+Rank [5/8]: Start tile-wise ring communication...
+Rank [5/8]: Starting iteration 1
+Rank [3/8]: Start tile-wise ring communication...
+Rank [3/8]: Starting iteration 1
+Rank [6/8]: Start tile-wise ring communication...
+Rank [6/8]: Starting iteration 1
+Rank [7/8]: Start tile-wise ring communication...
+Rank [7/8]: Starting iteration 1
+Rank [4/8]: Start tile-wise ring communication...
+Rank [4/8]: Starting iteration 1
+Rank [1/8]: Start tile-wise ring communication...
+Rank [1/8]: Starting iteration 1
+Rank [7/8]: Starting iteration 2
+Rank [6/8]: Starting iteration 2
+Rank [5/8]: Starting iteration 2
+Rank [4/8]: Starting iteration 2
+Rank [3/8]: Starting iteration 2
+Rank [2/8]: Starting iteration 2
+Rank [0/8]: Starting iteration 2
+Rank [1/8]: Starting iteration 2
+Rank [7/8]: Starting iteration 3
+Rank [6/8]: Starting iteration 3
+Rank [4/8]: Starting iteration 3
+Rank [5/8]: Starting iteration 3
+Rank [0/8]: Starting iteration 3
+Rank [2/8]: Starting iteration 3
+Rank [3/8]: Starting iteration 3
+Rank [1/8]: Starting iteration 3
+Rank [7/8]: Starting iteration 4
+Rank [2/8]: Starting iteration 4
+Rank [5/8]: Starting iteration 4
+Rank [4/8]: Starting iteration 4
+Rank [6/8]: Starting iteration 4
+Rank [1/8]: Starting iteration 4
+Rank [0/8]: Starting iteration 4
+Rank [3/8]: Starting iteration 4
+Rank [6/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [7/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [4419, 35355].
+Process-averaged run time: 0.9034384201513603 s
+Rank [3/8]: Local distance matrix has shape [4419, 35355].
+Rank [2/8]: Local distance matrix has shape [4419, 35355].
+Rank [4/8]: Local distance matrix has shape [4419, 35355].
+Rank [5/8]: Local distance matrix has shape [4419, 35355].
+Rank [1/8]: Local distance matrix has shape [4419, 35355].
+Rank [6/8]: Local distance matrix has shape [4419, 35355].
+Rank [7/8]: Local distance matrix has shape [4422, 35355].
+Run 6
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+ [OK]
+Rank [6/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [3/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [4/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [4422, 18]...
+ [OK]
+Rank [0/8]: Local data chunk has shape [4419, 18]...
+Start distance calculations...
+ [OK]
+Rank [5/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [4419, 18]...
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Rank [0/8]: Calculate diagonal blocks...
+Rank [6/8]: Calculate diagonal blocks...
+Rank [1/8]: Calculate diagonal blocks...
+Rank [2/8]: Calculate diagonal blocks...
+Rank [5/8]: Calculate diagonal blocks...
+Rank [3/8]: Calculate diagonal blocks...
+Rank [7/8]: Calculate diagonal blocks...
+Rank [4/8]: Calculate diagonal blocks...
+Rank [2/8]: Start tile-wise ring communication...
+Rank [2/8]: Starting iteration 1
+Rank [4/8]: Start tile-wise ring communication...
+Rank [4/8]: Starting iteration 1
+Rank [6/8]: Start tile-wise ring communication...
+Rank [6/8]: Starting iteration 1
+Rank [3/8]: Start tile-wise ring communication...
+Rank [3/8]: Starting iteration 1
+Rank [1/8]: Start tile-wise ring communication...
+Rank [1/8]: Starting iteration 1
+Rank [0/8]: Start tile-wise ring communication...
+Rank [0/8]: Starting iteration 1
+Rank [5/8]: Start tile-wise ring communication...
+Rank [5/8]: Starting iteration 1
+Rank [7/8]: Start tile-wise ring communication...
+Rank [7/8]: Starting iteration 1
+Rank [7/8]: Starting iteration 2
+Rank [6/8]: Starting iteration 2
+Rank [5/8]: Starting iteration 2
+Rank [4/8]: Starting iteration 2
+Rank [3/8]: Starting iteration 2
+Rank [2/8]: Starting iteration 2
+Rank [0/8]: Starting iteration 2
+Rank [1/8]: Starting iteration 2
+Rank [7/8]: Starting iteration 3
+Rank [6/8]: Starting iteration 3
+Rank [4/8]: Starting iteration 3
+Rank [5/8]: Starting iteration 3
+Rank [0/8]: Starting iteration 3
+Rank [2/8]: Starting iteration 3
+Rank [3/8]: Starting iteration 3
+Rank [1/8]: Starting iteration 3
+Rank [7/8]: Starting iteration 4
+Rank [2/8]: Starting iteration 4
+Rank [5/8]: Starting iteration 4
+Rank [6/8]: Starting iteration 4
+Rank [4/8]: Starting iteration 4
+Rank [1/8]: Starting iteration 4
+Rank [0/8]: Starting iteration 4
+Rank [3/8]: Starting iteration 4
+Rank [6/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [7/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [4419, 35355].
+Process-averaged run time: 0.9012156936805695 s
+Rank [3/8]: Local distance matrix has shape [4419, 35355].
+Rank [6/8]: Local distance matrix has shape [4419, 35355].
+Rank [1/8]: Local distance matrix has shape [4419, 35355].
+Rank [2/8]: Local distance matrix has shape [4419, 35355].
+Rank [5/8]: Local distance matrix has shape [4419, 35355].
+Rank [7/8]: Local distance matrix has shape [4422, 35355].
+Rank [4/8]: Local distance matrix has shape [4419, 35355].
+Run 7
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+ [OK]
+Rank [4/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [0/8]: Local data chunk has shape [4419, 18]...
+Start distance calculations...
+ [OK]
+Rank [6/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [5/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [4422, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [3/8]: Local data chunk has shape [4419, 18]...
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Rank [0/8]: Calculate diagonal blocks...
+Rank [5/8]: Calculate diagonal blocks...
+Rank [2/8]: Calculate diagonal blocks...
+Rank [3/8]: Calculate diagonal blocks...
+Rank [4/8]: Calculate diagonal blocks...
+Rank [7/8]: Calculate diagonal blocks...
+Rank [1/8]: Calculate diagonal blocks...
+Rank [6/8]: Calculate diagonal blocks...
+Rank [2/8]: Start tile-wise ring communication...
+Rank [2/8]: Starting iteration 1
+Rank [1/8]: Start tile-wise ring communication...
+Rank [1/8]: Starting iteration 1
+Rank [6/8]: Start tile-wise ring communication...
+Rank [6/8]: Starting iteration 1
+Rank [3/8]: Start tile-wise ring communication...
+Rank [3/8]: Starting iteration 1
+Rank [7/8]: Start tile-wise ring communication...
+Rank [7/8]: Starting iteration 1
+Rank [5/8]: Start tile-wise ring communication...
+Rank [5/8]: Starting iteration 1
+Rank [4/8]: Start tile-wise ring communication...
+Rank [4/8]: Starting iteration 1
+Rank [0/8]: Start tile-wise ring communication...
+Rank [0/8]: Starting iteration 1
+Rank [7/8]: Starting iteration 2
+Rank [6/8]: Starting iteration 2
+Rank [5/8]: Starting iteration 2
+Rank [4/8]: Starting iteration 2
+Rank [3/8]: Starting iteration 2
+Rank [2/8]: Starting iteration 2
+Rank [0/8]: Starting iteration 2
+Rank [1/8]: Starting iteration 2
+Rank [7/8]: Starting iteration 3
+Rank [6/8]: Starting iteration 3
+Rank [4/8]: Starting iteration 3
+Rank [5/8]: Starting iteration 3
+Rank [0/8]: Starting iteration 3
+Rank [2/8]: Starting iteration 3
+Rank [3/8]: Starting iteration 3
+Rank [1/8]: Starting iteration 3
+Rank [7/8]: Starting iteration 4
+Rank [2/8]: Starting iteration 4
+Rank [5/8]: Starting iteration 4
+Rank [4/8]: Starting iteration 4
+Rank [6/8]: Starting iteration 4
+Rank [1/8]: Starting iteration 4
+Rank [0/8]: Starting iteration 4
+Rank [3/8]: Starting iteration 4
+Rank [6/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [7/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [4419, 35355].
+Process-averaged run time: 0.9020667693112046 s
+Rank [3/8]: Local distance matrix has shape [4419, 35355].
+Rank [4/8]: Local distance matrix has shape [4419, 35355].
+Rank [2/8]: Local distance matrix has shape [4419, 35355].
+Rank [5/8]: Local distance matrix has shape [4419, 35355].
+Rank [7/8]: Local distance matrix has shape [4422, 35355].
+Rank [1/8]: Local distance matrix has shape [4419, 35355].
+Rank [6/8]: Local distance matrix has shape [4419, 35355].
+Run 8
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+ [OK]
+Rank [4/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [0/8]: Local data chunk has shape [4419, 18]...
+Start distance calculations...
+ [OK]
+Rank [2/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [3/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [4422, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [5/8]: Local data chunk has shape [4419, 18]...
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Rank [0/8]: Calculate diagonal blocks...
+Rank [6/8]: Calculate diagonal blocks...
+Rank [2/8]: Calculate diagonal blocks...
+Rank [5/8]: Calculate diagonal blocks...
+Rank [1/8]: Calculate diagonal blocks...
+Rank [3/8]: Calculate diagonal blocks...
+Rank [7/8]: Calculate diagonal blocks...
+Rank [4/8]: Calculate diagonal blocks...
+Rank [6/8]: Start tile-wise ring communication...
+Rank [6/8]: Starting iteration 1
+Rank [2/8]: Start tile-wise ring communication...
+Rank [2/8]: Starting iteration 1
+Rank [0/8]: Start tile-wise ring communication...
+Rank [0/8]: Starting iteration 1
+Rank [5/8]: Start tile-wise ring communication...
+Rank [5/8]: Starting iteration 1
+Rank [1/8]: Start tile-wise ring communication...
+Rank [1/8]: Starting iteration 1
+Rank [7/8]: Start tile-wise ring communication...
+Rank [7/8]: Starting iteration 1
+Rank [4/8]: Start tile-wise ring communication...
+Rank [4/8]: Starting iteration 1
+Rank [3/8]: Start tile-wise ring communication...
+Rank [3/8]: Starting iteration 1
+Rank [7/8]: Starting iteration 2
+Rank [6/8]: Starting iteration 2
+Rank [5/8]: Starting iteration 2
+Rank [4/8]: Starting iteration 2
+Rank [3/8]: Starting iteration 2
+Rank [2/8]: Starting iteration 2
+Rank [0/8]: Starting iteration 2
+Rank [1/8]: Starting iteration 2
+Rank [7/8]: Starting iteration 3
+Rank [6/8]: Starting iteration 3
+Rank [4/8]: Starting iteration 3
+Rank [5/8]: Starting iteration 3
+Rank [0/8]: Starting iteration 3
+Rank [2/8]: Starting iteration 3
+Rank [3/8]: Starting iteration 3
+Rank [1/8]: Starting iteration 3
+Rank [7/8]: Starting iteration 4
+Rank [2/8]: Starting iteration 4
+Rank [5/8]: Starting iteration 4
+Rank [6/8]: Starting iteration 4
+Rank [4/8]: Starting iteration 4
+Rank [1/8]: Starting iteration 4
+Rank [0/8]: Starting iteration 4
+Rank [3/8]: Starting iteration 4
+Rank [6/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [7/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [4419, 35355].
+Process-averaged run time: 0.9112897163722664 s
+Rank [3/8]: Local distance matrix has shape [4419, 35355].
+Rank [6/8]: Local distance matrix has shape [4419, 35355].
+Rank [5/8]: Local distance matrix has shape [4419, 35355].
+Rank [2/8]: Local distance matrix has shape [4419, 35355].
+Rank [7/8]: Local distance matrix has shape [4422, 35355].
+Rank [4/8]: Local distance matrix has shape [4419, 35355].
+Rank [1/8]: Local distance matrix has shape [4419, 35355].
+Run 9
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+ [OK]
+Rank [0/8]: Local data chunk has shape [4419, 18]...
+Start distance calculations...
+ [OK]
+Rank [4/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [3/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [5/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [4422, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [4419, 18]...
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Rank [0/8]: Calculate diagonal blocks...
+Rank [2/8]: Calculate diagonal blocks...
+Rank [5/8]: Calculate diagonal blocks...
+Rank [1/8]: Calculate diagonal blocks...
+Rank [4/8]: Calculate diagonal blocks...
+Rank [3/8]: Calculate diagonal blocks...
+Rank [7/8]: Calculate diagonal blocks...
+Rank [6/8]: Calculate diagonal blocks...
+Rank [3/8]: Start tile-wise ring communication...
+Rank [3/8]: Starting iteration 1
+Rank [6/8]: Start tile-wise ring communication...
+Rank [6/8]: Starting iteration 1
+Rank [2/8]: Start tile-wise ring communication...
+Rank [2/8]: Starting iteration 1
+Rank [0/8]: Start tile-wise ring communication...
+Rank [0/8]: Starting iteration 1
+Rank [4/8]: Start tile-wise ring communication...
+Rank [4/8]: Starting iteration 1
+Rank [1/8]: Start tile-wise ring communication...
+Rank [1/8]: Starting iteration 1
+Rank [7/8]: Start tile-wise ring communication...
+Rank [7/8]: Starting iteration 1
+Rank [5/8]: Start tile-wise ring communication...
+Rank [5/8]: Starting iteration 1
+Rank [7/8]: Starting iteration 2
+Rank [6/8]: Starting iteration 2
+Rank [5/8]: Starting iteration 2
+Rank [4/8]: Starting iteration 2
+Rank [3/8]: Starting iteration 2
+Rank [2/8]: Starting iteration 2
+Rank [0/8]: Starting iteration 2
+Rank [1/8]: Starting iteration 2
+Rank [7/8]: Starting iteration 3
+Rank [6/8]: Starting iteration 3
+Rank [4/8]: Starting iteration 3
+Rank [5/8]: Starting iteration 3
+Rank [0/8]: Starting iteration 3
+Rank [2/8]: Starting iteration 3
+Rank [3/8]: Starting iteration 3
+Rank [1/8]: Starting iteration 3
+Rank [7/8]: Starting iteration 4
+Rank [2/8]: Starting iteration 4
+Rank [5/8]: Starting iteration 4
+Rank [4/8]: Starting iteration 4
+Rank [6/8]: Starting iteration 4
+Rank [1/8]: Starting iteration 4
+Rank [0/8]: Starting iteration 4
+Rank [3/8]: Starting iteration 4
+Rank [6/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [7/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [4419, 35355].
+Process-averaged run time: 0.9022921589203179 s
+Rank [3/8]: Local distance matrix has shape [4419, 35355].
+Rank [2/8]: Local distance matrix has shape [4419, 35355].
+Rank [4/8]: Local distance matrix has shape [4419, 35355].
+Rank [7/8]: Local distance matrix has shape [4422, 35355].
+Rank [1/8]: Local distance matrix has shape [4419, 35355].
+Rank [6/8]: Local distance matrix has shape [4419, 35355].
+Rank [5/8]: Local distance matrix has shape [4419, 35355].
+Run 10
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+ [OK]
+Rank [0/8]: Local data chunk has shape [4419, 18]...
+Start distance calculations...
+ [OK]
+Rank [6/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [4/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [3/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [5/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [4422, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [2/8]: Local data chunk has shape [4419, 18]...
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Rank [0/8]: Calculate diagonal blocks...
+Rank [3/8]: Calculate diagonal blocks...
+Rank [5/8]: Calculate diagonal blocks...
+Rank [6/8]: Calculate diagonal blocks...
+Rank [1/8]: Calculate diagonal blocks...
+Rank [2/8]: Calculate diagonal blocks...
+Rank [7/8]: Calculate diagonal blocks...
+Rank [4/8]: Calculate diagonal blocks...
+Rank [2/8]: Start tile-wise ring communication...
+Rank [2/8]: Starting iteration 1
+Rank [7/8]: Start tile-wise ring communication...
+Rank [7/8]: Starting iteration 1
+Rank [6/8]: Start tile-wise ring communication...
+Rank [6/8]: Starting iteration 1
+Rank [1/8]: Start tile-wise ring communication...
+Rank [1/8]: Starting iteration 1
+Rank [0/8]: Start tile-wise ring communication...
+Rank [0/8]: Starting iteration 1
+Rank [5/8]: Start tile-wise ring communication...
+Rank [5/8]: Starting iteration 1
+Rank [3/8]: Start tile-wise ring communication...
+Rank [3/8]: Starting iteration 1
+Rank [4/8]: Start tile-wise ring communication...
+Rank [4/8]: Starting iteration 1
+Rank [7/8]: Starting iteration 2
+Rank [6/8]: Starting iteration 2
+Rank [5/8]: Starting iteration 2
+Rank [4/8]: Starting iteration 2
+Rank [3/8]: Starting iteration 2
+Rank [2/8]: Starting iteration 2
+Rank [0/8]: Starting iteration 2
+Rank [1/8]: Starting iteration 2
+Rank [7/8]: Starting iteration 3
+Rank [6/8]: Starting iteration 3
+Rank [4/8]: Starting iteration 3
+Rank [5/8]: Starting iteration 3
+Rank [0/8]: Starting iteration 3
+Rank [2/8]: Starting iteration 3
+Rank [3/8]: Starting iteration 3
+Rank [1/8]: Starting iteration 3
+Rank [7/8]: Starting iteration 4
+Rank [2/8]: Starting iteration 4
+Rank [5/8]: Starting iteration 4
+Rank [6/8]: Starting iteration 4
+Rank [4/8]: Starting iteration 4
+Rank [1/8]: Starting iteration 4
+Rank [0/8]: Starting iteration 4
+Rank [3/8]: Starting iteration 4
+Rank [6/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [7/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [4419, 35355].
+Process-averaged run time: 0.9093053898541257 s
+Rank [3/8]: Local distance matrix has shape [4419, 35355].
+Rank [4/8]: Local distance matrix has shape [4419, 35355].
+Rank [5/8]: Local distance matrix has shape [4419, 35355].
+Rank [6/8]: Local distance matrix has shape [4419, 35355].
+Rank [7/8]: Local distance matrix has shape [4422, 35355].
+Rank [1/8]: Local distance matrix has shape [4419, 35355].
+Rank [2/8]: Local distance matrix has shape [4419, 35355].
+Run 11
+######################
+# Pairwise distances #
+######################
+COMM_WORLD size is 8.
+Loading data... /pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/SUSY_50k.h5[data]
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+Overall number of samples considered: 35355
+n^2/p = 156247003.125
+ [OK]
+Rank [0/8]: Local data chunk has shape [4419, 18]...
+Start distance calculations...
+ [OK]
+Rank [2/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [4/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [6/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [3/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [7/8]: Local data chunk has shape [4422, 18]...
+ [OK]
+Rank [5/8]: Local data chunk has shape [4419, 18]...
+ [OK]
+Rank [1/8]: Local data chunk has shape [4419, 18]...
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Overall number of samples is 35355.
+Rank [0/8]: Calculate diagonal blocks...
+Rank [2/8]: Calculate diagonal blocks...
+Rank [6/8]: Calculate diagonal blocks...
+Rank [5/8]: Calculate diagonal blocks...
+Rank [1/8]: Calculate diagonal blocks...
+Rank [3/8]: Calculate diagonal blocks...
+Rank [4/8]: Calculate diagonal blocks...
+Rank [7/8]: Calculate diagonal blocks...
+Rank [2/8]: Start tile-wise ring communication...
+Rank [2/8]: Starting iteration 1
+Rank [6/8]: Start tile-wise ring communication...
+Rank [6/8]: Starting iteration 1
+Rank [1/8]: Start tile-wise ring communication...
+Rank [1/8]: Starting iteration 1
+Rank [7/8]: Start tile-wise ring communication...
+Rank [7/8]: Starting iteration 1
+Rank [5/8]: Start tile-wise ring communication...
+Rank [5/8]: Starting iteration 1
+Rank [3/8]: Start tile-wise ring communication...
+Rank [3/8]: Starting iteration 1
+Rank [0/8]: Start tile-wise ring communication...
+Rank [0/8]: Starting iteration 1
+Rank [4/8]: Start tile-wise ring communication...
+Rank [4/8]: Starting iteration 1
+Rank [7/8]: Starting iteration 2
+Rank [6/8]: Starting iteration 2
+Rank [5/8]: Starting iteration 2
+Rank [4/8]: Starting iteration 2
+Rank [3/8]: Starting iteration 2
+Rank [2/8]: Starting iteration 2
+Rank [0/8]: Starting iteration 2
+Rank [1/8]: Starting iteration 2
+Rank [7/8]: Starting iteration 3
+Rank [6/8]: Starting iteration 3
+Rank [4/8]: Starting iteration 3
+Rank [5/8]: Starting iteration 3
+Rank [0/8]: Starting iteration 3
+Rank [2/8]: Starting iteration 3
+Rank [3/8]: Starting iteration 3
+Rank [1/8]: Starting iteration 3
+Rank [7/8]: Starting iteration 4
+Rank [2/8]: Starting iteration 4
+Rank [5/8]: Starting iteration 4
+Rank [4/8]: Starting iteration 4
+Rank [6/8]: Starting iteration 4
+Rank [1/8]: Starting iteration 4
+Rank [0/8]: Starting iteration 4
+Rank [3/8]: Starting iteration 4
+Rank [6/8]: [DONE]
+Rank [5/8]: [DONE]
+Rank [4/8]: [DONE]
+Rank [7/8]: [DONE]
+Rank [2/8]: [DONE]
+Rank [1/8]: [DONE]
+Rank [0/8]: [DONE]
+Rank [3/8]: [DONE]
+Rank [0/8]: Local distance matrix has shape [4419, 35355].
+Process-averaged run time: 0.913113493588753 s
+Rank [3/8]: Local distance matrix has shape [4419, 35355].
+Rank [4/8]: Local distance matrix has shape [4419, 35355].
+Rank [2/8]: Local distance matrix has shape [4419, 35355].
+Rank [6/8]: Local distance matrix has shape [4419, 35355].
+Rank [5/8]: Local distance matrix has shape [4419, 35355].
+Rank [1/8]: Local distance matrix has shape [4419, 35355].
+Rank [7/8]: Local distance matrix has shape [4422, 35355].
+
+============================= JOB FEEDBACK =============================
+
+NodeName=uc2n[186,188-190,208,215-217]
+Job ID: 24894098
+Cluster: uc2
+User/Group: ku4408/scc
+State: COMPLETED (exit code 0)
+Nodes: 8
+Cores per node: 80
+CPU Utilized: 00:04:32
+CPU Efficiency: 0.41% of 18:18:40 core-walltime
+Job Wall-clock time: 00:01:43
+Memory Utilized: 8.45 GB (estimated maximum)
+Memory Efficiency: 1.20% of 703.12 GB (1.10 GB/core)
diff --git a/2_psrs_cdist/solutions/A1/weak_scaling_symmetric/weak_scaling_symmetric.pdf b/2_psrs_cdist/solutions/A1/weak_scaling_symmetric/weak_scaling_symmetric.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..f9b7151805a4cb835e2760685c3b546d7dd4794a
GIT binary patch
literal 18377
zcmdVC2RzkZ{6C&#T`Mc8=*pgVUn68kC_A%kk?oQZg{%_UBfG3nWMzbGrO+T-%P2)s
zH2u!|qI~fE)&Kwa{~wRv?|wd~bMBe1^E$6{UaxbWuZu@hQCS2fiiPnM4Z{m-U??~e
z?rdQXlahiXbiG}y;RpqDg1LjUEgYd~ZforhM}rF5a2XkxwUZT8QG8tkWoIV>97C*t
zFgRvmZ*56{i>-WB@**78A(#`a;n<Z7O>+Xl+RX`$gTBHLI;YI7?3`@j_|;N3XG<Mx
z0^Ar3tEdb}vGyXs5vq=W1o_oiVKr8To30}VwRr*%-4OL|)_}Sd@)6qB?#>==mS8>*
z{n~JZhP9QQxxBL%7!e8n6USrW7(5bg3PUJ>Nr9H`aEX<ga!yXp;2R36ww@7C@fzpU
zt(|NMr{Jj7(Uk2Rz+B-7We30jMQclED{F{kcY>R>xg*RgwIW5sc}RI@$m|amUv;_@
z?;rW^f6q(1CzC8@_Y`MGR{Wc%0dL?Zao=XYiypG&xyWeA5c19i=~Mm5MdBdl?c3Ig
zvD(iQ?+b-KKB%?FnG`qZ8J%sKJ)G6T_d^CIa^l6JWsXZlady>`@5yDH%(7a=x#fZz
zL-rn2{m$~8D)%Jgvu{^CEo9eulU=U-oVz0Q1`i_5{rzXZbB<<k6S2_|J-<9epxW;J
zIlO0O1*m@B+WRrjYiFU_{X))<dl=3#8inA~(J#m0as>^`35GSk`m;C0UaIBg@6^>-
z)YL3$R#4ZG>{HF)D!jREvGiNuvY=~LCSx2SypG5E)6a5(Wb$~6kFlxg@xqf8J^Dtj
ze#VCa-d%lr*DE*U68+0>9-rrHd~|NT6BZUMQIQ)oOYQfg39_cUB#_oD%-DIIW*}d$
zeoR&P6l&LCjBaNL(m~fIMZwZ&8cFXMNU$nCuJ|>9kEHYx38!#lQl`#Svstc$Y;HUq
zDJOTxxL<@>jI%f6l{>q47BW6I@^=r_#V1W{OV*d9X|WiPc`NDrJoZWGq~el4HMgY;
zFKMrGu58fG%alJqT7F-gC|o)##33{0P%qL}%>YyOv5vho)3Vzk&vo>;n?A43+ZpFt
zep5}32Zx!Ni>o5#FUDSp$5Mal*(1QsJ4;L26yYAmY(yKQZ5K!tgFA8GD)lz!0GDRM
zgQJdcx=Fhd&6k?wuY~zc3WK%zHRSFcRC^>ic{XgClDP^Smvr4tHP|ALrp-|Ejy`pc
zW=bIAWg1@cTNYtYBw%)q=KSK?Ic2;TS~8KZ_VEYnlji8AoR?RO=`fhp3;RYHSAu1p
z&5;kel%T*CQI_dayz597maLUapT(nxIq=YS>uS<^N2$2hQu~sJY|J*Pq&YqInQ=#S
zG#L*Zxv)s#!NndBlMoWP9nbR`U$2wox{LYdGkJkH-pov!IGG)+74$Od{nFa;!N>G#
zI#RA{nGKkT+1YXraCvkO?WZ4`t=wD6IGOKl@{lx7Ho|HjPoa>>>!SnQfn7r*A@QH%
z)45>`>;+RLl!eF(9MT!uMNi<=`h&SOofA2=@;8qRninrBk)2Mxz_D<(y`W^mEhpPx
zy5Ghjb8-8i$LamnBg>VgufKVO9qvaJS6O_nWkkp<j&{V8e^&m)MQHjq_bpS|bu{XZ
zv5@ri7mBDos$|mfoKNvwqvCCIQ$zb-Fv83-q6Bjm!m8-Jo|b-RP+1h<KhpO0TixvN
z*jys~(83Ec{tSBwMuEc|W+xx1RQh5V@2J!3`Gig|U4xDM$ThT)X!hQB_0{vK>#*h<
z7XE(UjPb|Q{3ngOT+yl8RwtYWlTS$GSnCGRn<v>FJ&Z5L_C~+JaWkch_ek(D-qwEC
zVT<a*Qg>c|Y|g#yyf(j#>rMBg+)FOV@NUZ_yInD=IaJ&n!)JY7T1`f%)G7SB*usbI
z!cugWgZdnuO%@1g$4Yl)?p`lDzi|wkO-hgz*a?i+jZ$CzQh&l8I^P>D*-w~^xSn?`
zI(EYLAWQ|vzjNS4RYo5B9>RM!D!a=OykEU`6bU|P58iz$n1jufKA2+%=HWH1QywZu
zDg;ES^sEjPx>uo04TY+$YCbf}{gki2$>fU-vg6%z^Vv0lr@1|u`U#zqBkA)P+Y{6!
za(+TXH>xe0)jvIFHe$c3?Rb$k>#8gZvmaW6;n3;(WjpppVfW~2+nHgKY;M0WlD;bM
z7(=^f-<vqKQR~@c_se9@Gg)K8Bg7O&t#iE{V8xLd1^CvsQ;56h5cRInL;an`d*79x
z=fHowAmiD1x9;v;<1t5$zGZ2Wg`VaH)+yzWVXjx{@3d=FjJ;xcNSW6v<VF{fm^1w7
zG{$L??0$#kj{;iVZoV7M?|=L{`|<)m&dA;EVxZ`?7W3E!^^Vw{14kT7CX0&m3WQZ?
zTd$P6`BYOW?bpC$?>>Njh*!$HMc>UYO6VXJv~;=o+()+J-AOtD^_w3{;$C-!IZI^U
zY1|t`D#`Qa6hYu1<|1{}YfD+97YbcJU$eFPC+k%%cx|uq9QvTXZGQJFgM#BnBKeC%
zHN>v<cu4vbB@X>KDHyMzdv183a<77aY5gO_WvbeTU!|<>pZtR34RypXDb90rhwt_K
zVsyNVS)@|!X3H#fp#mqHE;ew4>sw#ENKRGpce2G?-=sJi{YkxrK8M1QZ^jF|9+dCV
zDI5R(q9*F1s#s}aU7t6<yB@oSqopy|fD!+!xGBBb5)L^%U*m6hwSBRk^{2st&{JU%
zgXD{|O1jBy=V}LjcywdD(rT33ZXABB&l+&6;rjh>*D#9`+Gv*{L!D!3hdvEVzw9_t
ze%+tv{_K*qvE<Zo#on)985-ofBbTP74_Re3``ffc72g^q_&N6Kvv3`}SNH)FdZzh}
zyOngor-5aMpFa;*V@7qCNoV9)>LEj8Ra35-97MGWsaljc8nb4H5cM&XX$+M1TE#7N
zvN#6l;lJo(y@$25jY!HJJ?R3wN#^Lkgl98*E#r6dxIJz>XzFT8&6v_MtuEd&XBJk6
z%}UOZdPcaQIDKu<tKtyL?Yxnff>{-%N*~<^$CN^;AMKU1+IMa99MF*Yu)$mq!~dN$
z>C5*R&dXAgF6%}!?o_Pv-`7sjzud1teyEe2-0emX$|`)8JTc((wfI<G0amhs7gsXu
z`&gAy4~nqb?iR~A$I_b-#8Gpy<0I?x7BsE5omNd4GzRq#L}{wJH6v+izq6d(-m2#l
zbuew;@bXD&Wb(dKjW5YdjFsN(mPw;LIA12(&6UM-E-^z|n5NX^z|P7y_1Vnlc(wA2
z_ozSXJTLK9GkV}j?A@e$al-<sDeb|&VT;zrq0-HFK7PI7b+f<DNQWW$r6ZPJ6MN(P
zSpx#ygKeVCCh>#1`W)J&IhlgT8eE63klDke8DxVn(jDC8J_^oS&PQ(UqOVb1a%-m^
zt`!j{6~vG|durHc;cAm-$JNZG<LQsI?;N~D^Mt0XdAl_JkqEVPeedO~lFW1DADrtN
zj=u=@UnY5NsXx92zaaB&!y2NoxWDsDNlU_M2a4w1Pne(Cws++})2Vg|;Fp~h?D<rb
znqv61PZ*)CLrvalt64vbL$KFtGaoy|^)y9ET}y}xdn#m@3;$d{9Md{;?@iJ9>YS2h
zA)mIW((Um*T)MPpUU0mW9`2vCPXAEim(EuBL@*&wFFZr~VM#JOp(!~^>_(4>-t;d-
zw{*R)pgreR$Jdg1XL8wU0+JffI(A-Yw~KK(HrC?%+T`JF&496#JDrmGy0Walyp+4`
z%UgF?)l`)zL|?|}F$YnaXnpc=I$NRkf#+J3(9m;li)6L$C;Y-j)zNRH=c4H6_KyAh
z)t&V9@{e05PwJZ1ZNWY0qOg&B7_q-|52vN(M24aqnq;nTj9ub$Pd_g0(K~w1SV7m0
zTgohg^Q-ud7hVNi#aDQi9^-n*8iIZ}{3v2p2;=%9cj^Fo=Y-d-xjs$(<j>qEdFvC;
zBy@=LshcTPv>Tuf&B2nQoy7*7D&%rcg>a;CJS@;ux_V4cFHM4dTJ}q(3%YHu@2x6J
z&H30-?1z{XdZD0;ms;I5P1L-|6nCF^c6rYaR=R@fNS)hUjwN@aD+%X<-^P2^me%KT
z%(^Uc&bc{Mf9Qy0>N?{qDVArFahHPd`q?ADu*xgd?K(D5r()T<dfY<Bm(xby`TG6Z
z*7I3UZwszL*QZTf6aNRUMX5QFqG%Kr-C?Hi^PkIRKB;RG0NeIzKiZZYc7<oYfN=kc
zig&>SLTrk{sE^XoaKrhriw_&#dCOdkcRp6E5fI8B9&I6dLZ@;#@!IDju9qWG<%Y8l
z+U#?}=?}4^`MxdbjI=ziFesvNq&&=cqWN~-wW;pUgx$@eBpuuIxw&F4?nv%aoK`}>
za|04ut{P5`fB!~yt7*iwuHT-8RI$zEDa{9pr;kT5-}~lsQ(un1y3jqvKK774vC))3
zJBJQsNo_VZv_57qai(2rsCDMf?etWAm5Cjycif*($s5(g{HhME`gwMnp`Xd!E!YNK
zGdHpg_b*4=@2%=@Mnc@Gfy`xKtJ=jBtEixrdaLog!*h%ZgNm<P5+kx}w$FyzjdR?X
zHzygO`%Fa@WFz0@LMy$UAy_~t_D=c7Ju+u7i*JROazh+E<!Q4+3&Uq72rBw{yRBy#
zRFi`<gI+&St>v-XdyDj7;d^5%F4GnrhNF)odYlqMaYL!c^W+jM=pugV$vrK1S9%uw
z;=;$So=2^xZkC*oicc!z3uLPQ+<hB!`{G0Gx&5zk{vi~n>n499Lndf?BJX)8Re5As
zrIzZMrqA|z(K6ZPE=;_+m|%Lu&@?~hN2<m%h4N8%Km6$Mub<S;SB(X?;30H3-@ro@
z<{xyHMwA>QDJSKQ*}6c!ZGm@Y2XyV}CzksoQgFVmHT!b*AnrB!jJuq9B=|$S<|C%S
z$0hHjckICR0|CsHq=s?FZj;NO)~G)6h-UaQ`30)9x{jcIx4t_pbF0MHxZ_-gOMXRt
zkQ4FOeNoh0itEcyO+of}x-<=%NqGv@%nuKF)=d2*i{!#Q-eN+~+r)+miDCb~l`wde
zK2oy;Ny4a{n*f`zA%_Wsn8?-_gjdUpvZ~Ou+$SlcIgd54qf=L+J5C-(^_X0X#zKzy
zDqD)e(|c??t7&dxw{geqYTJUM^{**w8d&@v#8gRJHHH*L2{f?OJ)22sX<U4B*YYg}
z&pdY(H8N$&j+FaY0kRA?&QGd`P8*15yMNnXf1#YWnt1^2MscwRSJ&vE)gf>xY45G*
zrmNNmvUi4rXej4CRqf3+o{qSD`&o*{x4qFH;d#<U{OE8sXZ5F8rYR07gq1YgQj}GU
zqJ~-edw7uw>!{#?ROSa_794es6T7^!d@baht-R!lJ6NwwDz|Z5Avs+o_+{L6MEz-|
z!I0dd6di$sBCy!=kn3Icu;J)}A(Ff?y65f(&3+8Zy>jg-I+%C15i2YC%lhrr7aT?;
zeq2FEoJU(~%MiniwM`@6-R^siY5yG7re<iCHq?6Jo|LsmR0PTM<>%LKlUF$Zv1&h8
zsVz9Xg-M8#K>mHvm9D5k99CGA364B3x8$CFG{**2y^KeH6`9GCcfuiSn@jRf?}%=8
zdH22}tnpghiw0DGVrSJ{R~5`1nJ1H-6h<&{+m51rtz5K|rS?HMRf=G}Y(=9Cq0d#F
z%74_b8g=-iJ;QgsX*bCiJ`ATjK4$%Rdw8bS%iit!(V<KGx${~ImYEM1#E;8RMG_8c
z>SlcV6!%!MyofgXu2)5Q|D(vO%<<8ly!%X<BJ+zM^U4U_9Xwfkh{TRY`P61MV74$n
z(Wrkgjd7aQv_Or#lMdcRY4*0vIh{HXJJ4{MpRYAQ>T0&i<Oy`E1_x60OnB;iOG&m9
zXNEZ@Wm-@-OgYSycd<vZqG+$k)1aDfxy!kl0n)zYS#<)e`P8N)l#4rO&5f^m`{~|!
zA@(Z3^W#%b#~Ep>VRf=tmkFzD)K4DAPcQp@`1rx8=qsC3tzn#GL|)F|i5o3~KfM_r
z1ljVjd8Wl}?@3o}i=Y!-cx~xMT`Jk`5E9QU`<~|YGxE<%3qnirx63}h-KLwNvd>m!
z5Vg=iE_Ap4K!~o{7Tnpw5JuttLCGp<5>L9?@38dqF;m~t-xGYE|5_ewtb7v0dqG{c
zw0W3rs>$;e!FM46`^9`Vge>SuqvI4>&LcF9mMdP5<y*oz)^w32J;HcNsa<KVq@@S7
z@ZKZT8NoqZ2Q8eA7iz%x$Mn8s1hhupw7$le<SxIvt20*l#=AwG^C$8h<F#N4bUfjF
zH!zdMMJ=y>kxx`MoV+twD1FG;QCxV7X>MV^VzB>kpyOgxofJ`&ii<L2tfxBFIUaqw
zroSugyYnabeVBubmY%rt{Wz?5fm&O%iB3J?`i<+yzD{?|CUl3rXS8Rm=Mm=SwH5OY
zxy8kxGaS47B}P)F`O8d~MXzsbBTfGee(4E|*o%)M{qG9KJNhK+b+iee{MlB(?K~f*
z&?k!Vjod42zD=!Uc)Zz)ue-n^&xi5m=i62(A0h|uq-F6ss-6gcr7~WR4X8anWO@8x
z6W6C%@;FuubLFk;nNs88dHW1=!VhLQeZ@G1SzOs(Y!fk>J;;kPVsOk-RM~!<!SHQb
zb;gG|lR85^b2fVcf7zzzkzv=g2O@KeLnB*<@qGhj=7$F?-MR9)%S$g^<mF~5ygz4Q
zXYMx8T*GtiTUq@P$Lgkn(=lNp#ykcgeO0{{S*mT1PltcbqxsBoTCWh(KHc^3aliv@
z<NaJD)a1NJf1PWxJsY)%=qvQ!v&9l^VI<@J;f8_1WAu>Q<W1GM$SmPQ!yVe3tydm&
zJl)sT^(K&N!i8;*!TI<%E^M-PNjLW0g!B<CpH6>^;kU3hQU9<HF!)kU<oRZ11<*4G
z8|fByEM^OX4241e!#Nx!N2kCEWG^+4i_)U^({rrp^Y6|FJC?^Th<8UECI9p&IOk<a
z-}6iJ#wyYus|u1oWtY$=3-S*upzmuSEZ=vu`m#rld25Tn*-dceu8sw<T`_&RX=qWs
zaQ~YfO}cIBj3y=qo@T186uUEpFT`_n48CB$__?oaDjVVV%MgxM--1nBSVm~%KUfkP
zT5?W1IB9A>!OofSFmC6Z9k73<@O}BTqmA<0PF~(ND!a(YVXuM4`O9qKh|@*lLJbE5
ze_}H}Mk#%5PH3K#U6K;_Y_Da<e>l?6^T4Tf`q~LG0c!ns{Cg9qj~<SF_cFOE|Mdkj
zp@U`$M`u1>NBhLEg`7!ptgoiiRy`zKQ`(+<?B@MlgxeZQs#9*rH@uz?mx^9mr18JM
zC(Ka6tIU$Ba8rJIK+8*mKT-d!I_GPepGLRQCesSn^vr5y8lQVP{~W0wnxyxRR8QZ6
zDO*@qD2&A4S6Di-7E&<^zQ^v7(M-MTmZAz|PW(Bfc_tD2g_I|HpWV*<tgE~s6lmV`
zL4Fp$l$+j<dsB7Nf|#qF-=C4w*e+f1B_Xp=P%7;TNmXF=+^I?-HJY{&%+uE)hjpT<
zqp06;+Wc~D4d#7{kMg*7ORMb>O=a15w|`zkV&R+o4#}$8ncBPc)1IQ;o9BdDKc^LV
zjEH$5<jvi!S5iv)stR()gcPmq&GkKW%$?jt<eeR?pjo@Q6BJIFyTOq#gu3}^8cp2H
z0AW(JcDHo1b0Ijp!BMLin2v`9fr!9C17To@qvnnP$NC@p2c5{T09Ya@q!<z|f(0F+
zB*3E*3MGbvkSvHm4ReB<9TDXcMIs?J2mbq=G6iTKpimQlZvfyTB4aRwk`utIAZYFn
zB)FCZLnzwW*jNK#4MNh4;n4ndcQLmFARoBLyE~Y>pMvATiYvjzz@c#zE)L4I;SwNi
z0Y_NE5r8dVMm7LT14o)2h(KLOIKl~ra0Wz!{R1#FcQ^tde;xo!17@%{m<9mO5XbGT
z2&Vvv&>^xOgCnwT&7%LaKI@oEWWrxK>0oZ_4(77Lu>UPj;Rrc*O9)n!Km*dDVW2M}
zXy{g?VD6$~ZD)Im04=z|S`vVk08X{DlykCmum;p4bO_dt`cOi{+>1CZI0}nJu17T2
zPD*1q9*O+_BEbIN_JhHS!?8F#90%4y93u|a77s%qk!ZNM7#fa2;z6MpG2qY`Vg-yi
z5{?C{1jQIU4r=f}1=#vBESM-1ph4CG4)g&AL&C+t9Zd|6hKpgapb&5bJn2F~oJf@z
z3<D7Zf*6o`Ja|x&kida@VsLRBXo~^$pcrfrG4KY92O=pB-jks|Q79w^Bv7CL41mVL
zpxHoFLoq}J4h#+jJm^>qh#F1;)CVyhNH-QJG$<_wI>TTw#DE8FK^m|K3q}A1cyY1S
zN`O_MF%D3M1&xViI5dzVEF2FW5b+?51C$Xv1D!xIAYf(PAQl25h^z&J;sJ3)wyaSH
z#Y7Q;=7$9>Q7F6^u?d*dn)twoD*`wQH75peG80=u9R8Dl2!y&oN&w2B5{N25<(ecw
z5<pxyAn{NPv1^Tx6{-M`jt4d+#6L(9{zED;6Dn9?E=&yS1q6sVs25`XY6oC(pw?^u
zK+-@I1V|V*1Qb#n2l@g7uS&*Rye1iIF*N>~WI$5!p8%CXc_=gtl907P6b?`(hLRu(
z0wfF@16Xh%EntBm$ygU)Ym%`RuL#77U_j~h0U~a#91?)F00{?>G#C{BK|0ny94LWf
zpLj5WBbSIXXhX*?*d0Vq2MaiQ)h)pi4z?*V&k7FKl$f%H10ew^7&w;EtBS}Dj$ZZR
zaD<~*{Uw~>;KU=ga0WYnCFKSOU;!}?IzCr=Sy2LD*RA9el>=BA;y2(cvT}ff;~lIz
zG0z)pU1I8whY8p`o1SST{^3mo7ePdE6n%5=Ff56mUj!)5mtT%yBCNyG)8U^S%&gKv
zxyoPdx}EEyqpp24{|t?Nju`vFDV~Qvv7J7mdlu+jKeJR`C5_SIjt*5!d=PK@=Hyu!
z8s~i$o9zOy;Wv@^FRzQ3+gLsDl2pEpF(hFEHldtAd`XfCi+@1I^x!|QNXPeVYlxuT
z!bHbzVWMM@|L}}19kco%EE6ocv+>(N`lsvsHA$oAhWL1<FYX)IrDUH_qi(3faRnQD
z;oS6`OKIBtL5FS|n{lJ;#PBz9lRYdJAF|I5bSaw}^0mIbU?W*Ir6s{if!DZ^Q7D`<
zBI-rg&zo-ih2c<LPbSAxoG6tT;rKOc0j%YX@8=%-{)j-2+pC*8XczkKmpY-SH#J3Z
zIQ)QRBJ1QUIjZR+O{a{);?F3N%6*tGzwb-+VQAF8<2FS>(xZa{HYM)}+M-UZp`08Y
z^Z9dcu0203-@Q~K#IvCCJOty$_u*9A#M6$KSALCN-k}zC^AzW&@mo%PqGokEl1dg<
zN8O6-F6kXz)MS2A-F%m3=Vu2c1p)WyXKzn7KcYUJ;IPZ<$Tu>MYa_gBbov8B`A27q
zr3QKy_Z#&S_HVK5YpNMCjn<y){^b&b(^7S6Ap&MU8n^X*sN-wR4E?g)uYI4HQkJLk
z4v&Q;*LDPINg`}`DF%0q*WcFd_z&S@!H1iQ$5ojF5?`F~41BP+&H4}>W`A9x{-f({
zu{0e@cxfpH^mCq_lE_+ekLNan;q{y^%DA*2DknVYXIywJxZ}P~<=Au}qhsMW_wPm2
zj^78Fj(YmORei%0HknY=X#I*Mw$*@KTB}+O<#*++Ll$3F9e+lUgN%KcVj{ooc=)}0
z{5^Ct)$eUz>T#Yj9m#XF%vV0r{-l9cLnDUrt0I+cCHBLvU+22-*7tv>Kgy}gum!`Q
zUA|_Y{bty|?B!z;G2o)~z<5skeJbwjnRC694?`xtf^dqgUoQJD)JICwnDxah2G#
z4;fj(ZN_!fTN}QV(=**z9OpO?cZu3OB-*8~=#^7?B<r$ge76C`EPm;t;fbBko7DS`
z@3IvgBD1thZIS47iZEp=z3fq(r(<v!nMHGfP}v*@ch|kX&>4xIm=cfXOTRSxWk}dl
z!k_LPHU2Uu%jA-vPar-hd&YZq`yMyBWz&%*4yD>D9{66WU42>>A5!03IC<BTz)``^
z=q{fkC(6Y6a9S{TXNqP~*|kh<EtdkBys4s$b8k+Xh-|^H_4kc6eqsOR4yAc8QjwG9
zokTEVj(eWVg+sYe#AbHvCTWU*rn-M6hP#I$;4lp;?$#@w;>?=?4VUCUI=y@s)C4b7
z(1t(qp${X><l>{5B-v9jhs?z+BO8@a<f+qyTxkkkiSJFY3+{WZXY^J0!fzaie2a+`
zNEh_Y@Y*H(z5{Q}7bE=|{z6+jyfY^w?*1Xi`*AQyo8BG5jMd{-Od8@3YE`u(*y7M{
zIdVM@_!UWYT$7x#(s)$Fl|1m|*iiD)K9{PKgH}i5+X)<9OR43Lf+SdK7GH(qDKst*
zPR%Sb*$E=FYuSy`{5=@sM|Mx{W=ym_$2MlQ{FUNYsL_EfIJjOhu5u6qteU_207z@A
zIkn0{D5FW{<GHKd-r9O2b>E#&D`*!8)`g?mkbd{CRKHO(WpoYhtU7ViSSt6dsIgJ9
zcUbME%RDjwBm2NvX-W_+6=y#@PmN9!RLp;Pqc>|wJ|URm%F($j>Lx#lU#}47@Aa3^
z-gzO7mMh_&5Rh+k(qc<tlw&ZTjAqHdu9;!jG^X|8-6ERI-Z>(N+H&6lRXLfwqe{#6
zx3b$5XX12=uSD8%Gc~Y`*jnr}o9w+!a+6J0F~N7HJ*?nSeng#Oi_9^NZ~O`BBKmDk
zI&3F@#6>(kOCS?0z|P0b_vn@NP#c7MGk%fyp|UjD=fD{v>C`tL{$@X>70vj_OijWp
zEAiFg@gYMGhMVlwcODS}_Q&_RsF%O<@A!P|7h%Zgp+}t?Wwz`a4Y|2mx|_<SEiGu7
zw95iM*}^4zQtFg3_sn-P^er}bknZ{ER<{$IcltoN!GWt47K)cII=D;U-O;B$+bYr5
z%+%yCo7R_g8-o$#3>>Tpv41D)Syunar846XW-8lpi#6Ti4Ilk4yFx=t^&%-ab|+cB
z$#R~Et;<tLR+wDwzfTt|v4iSXOl(4u0X049!hT`trD#rfv7dYWvj=JoePTKXmXjtA
z=x8lz=Zjj%n}+S}&SIF8&N|I#SbF!Pgz&Xw_j-!`!FP6GvPdlbRMFM>6Y-o(M`=9v
z(HUKDx^vp1`K3{>)a7BP@5(mTyccMrtmKDxpAUn-+kShua2y`bgf@Ic`kuCv*2t{C
zV6WDtK%uxkDnhn?%Z;w_vWY%VF$IzN@qE9J-%jZ#ng#c&eKznq87z}jknH&(M&9Rz
z#=!$!9I)%?gCar|zAbN^0=^x3pgs4qb8(6N7X8B?CaIe_2<+NT=P<DF|GFuZfcHrw
zoN~vqt{fE%Hh7FT7O1VuJJDy&Ch0z0AfvAIR^Cq*&wGN!fZ~Ydux6C(-4|{mox^1G
zE_~+(dvsG1o}96ZR@D<2zRu{qOlQk`CEvT(GHsFC@;lcSSPihCHc<&ZEd0%V2ZOHC
z1XdN6gfV0n?;1Xruv1@@Rb7~(YkO~k3+zDpy?F9%)wZ;JTQFw}dlLg}=D&`M)2elB
zMbgaR$XJnZf1Zc0;40Bo=dK<cEKqq;w_ooBI>$uRqf^k(d_G6P<6`dFber5Cbf2%h
zO5zyctkLO9@ZG&DA>8{2dufXBq6;;?ep?EoJN3s!$~mF)T8)aCP9GQPYYZ7>_b~rp
zo=Pc9FtFH8dnvehfaRtBh=9SJ!X;1cY6>bhl;M;Pozg{ts>ZWy!UNGpsqfqjVarZ_
z+xM6Ui_er@QXKbgBvt4BWp&=6TgG_i?OP?hXNdT8&-qK|uRi1^k<$?`l(}R6-M)5W
zGMS%GT%zw*HmaWX=!BZ~=<(|I?o7GSC5)8qnMh5#6iv#7k?ru%kw)bUG8Ns8qNJ6o
zwP!4kd`-0ysNaEyeaY8h2an@3UHZ<?f3#2Cx6yLA692h(n(qE^CX)q+vsISy?aJ?B
z^ksg_Bq!N?t+rT-E$r%lao|$=ZTg$-@aO#^qZQXQ+CaOL$LS8=h_neiRkDB+rdfeJ
zoklps)HPrs0ZDCgSTizh=9>Al`nNkSf6mqZG)o_sK2!8E<Q>U&`8_sUkhX>0{ZA%8
zz8nZA5{smWQ4Wz!hz^&($jPd*t%dk7f8LN>c2BlA#ZBH&p&%jJZ7<4cleQRo3$qiv
zXZ%$@F&MlKgj;WC%*_qieB0;}?g|j*qP~)_PgH^066t8+aNW$tg51gwj)K>q?>yd?
zl)^;11vOjPbQo~${3|&)pkX0Do^F9VvN?0d9(}rT<5E4)uuOMZ)RAVd1Q3Rg)#{(}
z4COSxHr~I;Qz}0i^**IbyLE9_8FRPDt;5|6RU)t*T1?Ug$q1GMPrDbUB=cR4WCR?i
ztYY3<7o@r^$bN}}BL@T5oRn>=wLoi=268`lw{xWocjFB|cfBrsjIa25)`GNQ@`JHh
zwVWPyjiXZ}t@m41Gfh>$W+o>cQH)7^=T|Df5YiYUC36_nd42{T?wC!+JIRs5)78Q(
zS<=3}&L-uV-jrDrN2ox=aq6;s4u^STruggZ#jn@~kfRTfIu=<kO30p0SXc*sc=HN(
zt10ZgR;+2W%ZG%AXDEA5{o1?h{<tMe%=yE4rHr(8;*DXA=_kZnPkdsMplb}XOiK}_
zu{_0_=Mx#;*pptCW8T)zVf6VEz3%kc&@C2X3u6%lZr*=g2&E`Bx*bGFB|EaQegC8@
zdOFyB`ThAcCnihjVBLV~z$|Ie)1+xE`#ISv0i==;sbnal>!Ynt-#H#y)b@aLI8%5o
zP*pI5Q}a^No@fzkr>gTqc-q^y<8^W(OD<hQMtv!FVBt8KZG6BZ=1@?JAY%Ac4SPpU
zU+rl2rTTN%Nl;W<%w{XY4Lpqgbv8Kd-_CXL^G-vv1K<??(Lh@;z)b2&(2hH3r0e-0
zG+4J{hVZpKevnb@^%?8n7OKud#`oGGD*1K#oX=*H(YLB_wmUksbDCaXf9={Cc&Iri
zkGlP9W$CRq9qdsY1}t1zijC;jmZ?1nk3N*x82cyKHAZ<|NEPZ6X*VF#-a8`x{^DYm
zx5#bsu)E0#7h&xB;k*3hK4u=i?Du82t^vIF{XSKfh;Ys5^PT~NDl`(l!k$|D7cl%N
zrm4ZbO2cviJTXZUtu=&&v8A5U%wNwV#2O?8r;UvaEg#G5eIM5*jGp!}SRj1rYY(Ev
z&Sw9Zd`82FwZ0tFMo~N{5t5y(rSDax5KH?I)z0~eF1?wJ>c}@K=7uO9Cib|FF*@3;
zb0;~jMWo_Kw@+jra%7>Pds)_eqB?-(2<z#XMUn#bsetL_TOTXx`}_B^zdyf)p9*d(
z3+!J`KM^feH&Qvu+HRqR3pROUmWe{4ub2BN?^BS|UXmpkz08crfD0e5dlOxFs(B29
z3LGN~^%v@QG<Yqm3s<$&#ASJY^jY9|?wUl^728)rV^c)`BSK(!s`Ukm&9<uF`svOY
z(_7NXa-BscEj&!D>d(x%Z0QaqqeaWzrN!Wr$_9z%SDf_sU>GB@!77}=H?CfPv_p<D
z;->Otyc_fP97Gf$MDKDwYQ|l}@npA#`01V*gQEG@aU7<uqI|4~qJ0aW@uaL;c182D
zDvdv#=s&yPjvez-n|!qE$t&N7o%2n)X02EIon-OPUXIm;z4{(;YjX1I57Lun#$lTU
z9Q?XylXZgvFPVQ8aE&Xf009M_e5p;W`4=#alTr;gYEI2I6f0181pIgncaf6nape{z
zSWQJl3SHgzs<8}i`Lb7hx$}-p3rcPzTf`vIvwfh8(Y*<c+Fq&LIUtqYNZ)Qk$!L|h
zzrGQEz%=Ts{wKrOh3=|@BT+TwHYcUC8^dQPTT4iepZ{6Y9-!9r^~96ciGn<qOri4u
zZx3kW4`%JG(4BF+aj|UVs*nrI!jE2s5AVq;xTMCmnD!P%9U43g|8?3pEeY_e9h#X%
zmS3{m5eTni+58WGH9U;y)#tcp5>g(=nLMu@o6q@1N5&^vG__{9zsnvwnNTj`e*R8n
z9ilj<G2A9B=$>8D9g-9&5{3TWd@Ti5j-IDEI{YQWJ7g|aInj?;Rs<wga!!WBE%@1l
z<sKF&G3|2f=DUt-IB{w3XyW^k4xOB~gxvw;B>Lj}`ltHm;-!!A_7YgoPYZDmc2hAa
z9gzAOnfr|Coe}min(=8HLy<jc-NB#8quaJIxjxu#m2o7l>~)KC=|UO#@o&FKG7jq|
zY~~L*;{J%m9|#M+4q}wo(9%-WUk5Ql%=?Wlf|sau=;8{9wGIFS@BGAv*8if50G%VE
zi<`RM2!erEWDzVL14oI0`vM9L)C&L&{|+pIpV9pW7O})f;{Qj$BIK0fV(n(<Yy}4|
z>j)ofH)lA)*$IG$gfk#TIOS#y0g2!zec)d^Pmp#8ASc4z+SA$zV2lt72SYg7fd&Ad
zK*#|Y2;gvV32>J%2k_7zED|6GaD)Sdn5?3X|A8C9gFM8#KQGfnUamc7{aLGZ++GJa
zZelIq+y8(xZpq~TKafTYcw1jLoi&yFe*n@5&}S^br?KFaBmyBAJWK+@?Evh928as=
zxa>gzLW!Vq45$y>GFM`NAV4V6|0clJ$}pfV6jqR+6|fNpPz)kWg%^h)Bna37a1SuB
z@!)9}2Vs9u0O$)^0tW*D$P~Z=A&e003o#&{Vi1x8VTn)-l>?Xq4}O>`2HHb?N{9pK
zNPHy^iiwlK0N@f#2qI(+K*IxvJg6-WOnjveP#26#Oal-F4bUYh;KU?|u>{Z_cnJ0a
zkkv{Z2)#oQ;aZFY)D8<!DG17f&?zt%C_tDX1i7s;jmT`kR3d{w3>XW8#>9a(|H1Ac
zB2ke1KqEm|Ez#+Kh%7@>gSuF0PFxeHIS4S^>YDxTY9#>#ex(ZlND*O1h$=uiw5*UA
ztS#4C45)>zaf#Ri7QnV(+AFdE<luMzYXU+PCeRb~{dX@AQAi|=Shm`M1n^f_3y>hJ
z$N(e<{|!W51G#`?VqGB010)k`fha4hG6BiM`T+5o_#N~C$;MiMW{yYVz#0NtAsN^r
zNB|&~C>KC=;@5>WnOKV<rhs*a5o0L5E)dsxr5vjBJ3w*)ttu4%Mi_~w_S#t|2PcAZ
z09RB6Anl4mF^3bOD3E6f5c8Eha9pqAdPJ8HaC*X4PGjg8C#oUhX${`)R=z+7IPq-?
z^x^;nW+j_=a03}6)`3oa9C1k4no7W};)yHAJ@BX@R)jPF*qFpL0ql|06j4<GUrvw@
zBWe$#Eepo;06br*LsTz7h@htHPDk*4ZT`vt5?}3584}$MRo6ca$f4Fd&iwurJVHSZ
zG(uRKgUdA-10je0AFBa)<WDus|5yzv3Un8!2*olo>j+=>TJUE!>n`noGS^=qHK6TH
zoEJg^?57nL|EWqWTVuzH>RbE8KjOD(q3rAi+<E?Z(7=Ib64;v1|G;WkeT~OsfsOH(
zALL;)J6@tS=U%>ijC+jH>%r-Dkh$HP*(aK>LYF9TJB-9&+dYb2a?N^Cb=iJq!<KCa
zw&S>iE<FgR-X^q*<?Uu8i#VPCYpGxVKDomBl|RjZP_wgw+_1p$0f{c;&E@O?{E>(f
zw)&Gu;5YL}j1|D&AAmg0_<>iLi-R-4!OjBii4m0$MGM1E5eP2sk_d$3?;=rWH(LQ1
z0(gg6d02u!XRyANi<J%B!rbyS7-h8wP?OlbowJi7^k*OV6(!L~G;k(CqLC6v;OZ%a
zMDrn$d;;qv)yvJ=1_qtaFyMu<`T^P%2hJiJ`0qSmoDs|5POEt+;Kc*BAoN+6hXNXV
zV;<z<zA+C29T?DOU0V$Bn%k5I%*Frafp5T^+?a=f+##UP@BWZz2%XrF2iO1(p^fz*
z_lp1Iq41D5*v33aGyFFXDTV`<%Emklun0Ef0fS`Y7)bEfE;h{#jRu-<Lt8W!aub0*
z>*j_7u2viK#5eH|i9vx2!-jeo9B`7|m?sXj*@iqU<Ttu84{|};n1|gw23CA?Jsb+!
zBGBj0JaCxJ>x;v0S`Q>1xtUjZ=-$4eKRg;ZE^W-iZssXo3{sKM=Qk}VtT?2KHss+U
zt+yc$4LR%lHxDPVNyd@l;=lv!zxBW#+$=*9s7+%aC2)}Q&VR;0N<fywfAdgC6l4<o
zryg*F+Qb9kNd_#P4fRlHNQM8mE$D6&4*+=^<p70}0G7*!wqR>*s)t4cOJ`#}%;s^?
zVw=_%1&q0kJOB@?_)Y7E!2|Pl!x$I|U|Da-!;6D!%<nvco4K8XwHxtI?&#S0fIBL<
rooYKfLkBm}dQo+<afTBQZlXQnPB3>P5RXc5!~sK?hezp{GVFf=@`r$)
literal 0
HcmV?d00001
diff --git a/2_psrs_cdist/solutions/A2/psrs.py b/2_psrs_cdist/solutions/A2/psrs.py
new file mode 100644
index 0000000..c3ad018
--- /dev/null
+++ b/2_psrs_cdist/solutions/A2/psrs.py
@@ -0,0 +1,266 @@
+"""Parallel Sorting by Regular Sampling"""
+import argparse
+import time
+
+import h5py
+import numpy
+import torch
+from mpi4py import MPI
+
+__mpi_type_mappings = {
+ torch.bool: MPI.BOOL,
+ torch.uint8: MPI.UNSIGNED_CHAR,
+ torch.int8: MPI.SIGNED_CHAR,
+ torch.int16: MPI.SHORT,
+ torch.int32: MPI.INT,
+ torch.int64: MPI.LONG,
+ torch.bfloat16: MPI.INT16_T,
+ torch.float16: MPI.INT16_T,
+ torch.float32: MPI.FLOAT,
+ torch.float64: MPI.DOUBLE,
+ torch.complex64: MPI.COMPLEX,
+ torch.complex128: MPI.DOUBLE_COMPLEX,
+}
+
+
+def sort(a: torch.Tensor, comm: MPI.Comm = MPI.COMM_WORLD) -> torch.Tensor:
+ """
+ Sort a's elements along given dimension in ascending order by their value.
+
+ The sorting is not stable which means that equal elements in the result may have different ordering than in
+ original array.
+
+ Parameters
+ ----------
+ a : torch.Tensor
+ The 1D input array to be sorted.
+ comm : MPI.Comm
+ The communicator to use.
+
+ Returns
+ -------
+ torch.Tensor
+ Sorted local results.
+ """
+ size, rank = comm.size, comm.rank
+
+ if size == 1:
+ local_sorted, _ = torch.sort(a)
+ return local_sorted
+
+ else:
+ ###########
+ # PHASE 1 #
+ ###########
+ # P is comm size, n is overall number of samples.
+ # Each rank sorts its local chunk and chooses P regular samples as representatives.
+ if rank == 0:
+ print(
+ "###########\n"
+ "# PHASE 1 #\n"
+ "###########"
+ )
+ local_sorted, local_indices = torch.sort(a)
+ print(f"Rank {rank}/{size}: Local sorting done...[OK]")
+
+ n_local = torch.numel(local_sorted) # Number of elements in local chunk.
+ print(f"Rank {rank}/{size}: Number of elements in local chunk is {n_local}.")
+ n_global = comm.allreduce(n_local, op=MPI.SUM)
+
+ # Each rank chooses P regular samples.
+ # For this, separate sorted tensor into P+1 equal-length partitions.
+ # Regular samples have indices 0, w, 2w,...,(P−1)w where w=n/P^2.
+ # Here: `size` = P
+ w = int(n_global / size ** 2)
+ partitions = [idx * w for idx in range(0, size)]
+ regular_samples_local = local_sorted[partitions]
+ print(
+ f"Rank {rank}/{size}: There are {len(partitions)} local regular samples: {regular_samples_local}"
+ )
+
+ # Root gathers regular samples.
+ # Each processor has drawn `size` regular samples and we have `size` processors overall.
+ num_regular_samples_global = size * size
+ if rank == 0:
+ print(f"Overall number of regular samples is {num_regular_samples_global}.")
+ # Initialize buffer to gather all regular samples on root.
+ regular_samples_global = torch.zeros(num_regular_samples_global, dtype=a.dtype)
+ comm.Gather(regular_samples_local, regular_samples_global, root=0)
+ if rank == 0:
+ print("On root: Regular samples gathered...[OK]")
+
+ ###########
+ # PHASE 2 #
+ ###########
+ # Root
+ # - sorts gathered regular samples,
+ # - chooses P-1 pivots at index positions P + rho, 2P + rho, ..., (P-1)P + rho with rho = P/2 -1, and
+ # - shares them with other processes.
+ if rank == 0:
+ print(
+ "###########\n"
+ "# PHASE 2 #\n"
+ "###########"
+ )
+ # Initialize buffer for sharing pivots later.
+ global_pivots = torch.zeros((size - 1,), dtype=local_sorted.dtype)
+ if rank == 0:
+ # Sort gathered regular samples on root.
+ sorted_regular_samples_global, _ = torch.sort(regular_samples_global)
+ print(f"On root: Regular samples are {sorted_regular_samples_global}.")
+ # Choose P-1 pivot indices (P = `size`).
+ rho = int(size/2) - 1
+ global_partitions = [
+ idx * size + rho for idx in range(1, size)
+ ]
+ global_pivots = sorted_regular_samples_global[global_partitions]
+ if len(global_partitions) == size - 1:
+ print(
+ f"On root: There are {len(global_partitions)} global pivots: {global_pivots}"
+ )
+ comm.Bcast(
+ global_pivots, root=0
+ ) # Broadcast copy of pivots to all processes from root.
+ if rank == 0:
+ print("Pivots broadcast to all processes...")
+ ###########
+ # PHASE 3 #
+ ###########
+ if rank == 0:
+ print(
+ "###########\n"
+ "# PHASE 3 #\n"
+ "###########\n"
+ "Each processor forms P disjunct partitions of locally sorted elements using pivots as splits."
+ )
+ # Each processor forms P disjunct partitions of locally sorted elements using P-1 pivots as splits.
+ # Initialize buffer to store partition mask of local samples.
+ # `lt_partitions` is a tensor of shape [P, n_local]. We use it to assign local samples to the partitions as
+ # split by the global pivots. This is done via a boolean mask.
+ lt_partitions = torch.empty((size, local_sorted.shape[0]), dtype=torch.int64)
+ last = torch.zeros_like(local_sorted, dtype=torch.int64)
+ # Iterate over all pivots and store index of first pivot greater than element's value.
+ if rank == 0:
+ print("Iterate over pivots to find index of first pivot > element's value.")
+
+ for idx, pivot in enumerate(global_pivots):
+ # torch.lt(input, other, *, out=None) computes `input < other` element-wise.
+ # Returns boolean tensor that is True where input is less than other and False elsewhere.
+ lt = torch.lt(local_sorted, pivot).int()
+ lt_partitions[idx] = lt - last
+ last = lt
+
+ lt_partitions[size - 1] = torch.ones_like(local_sorted, dtype=last.dtype) - last
+
+ # `lt_partitions` contains P elements: the first one encodes which elements in `local_sorted` are smaller than
+ # the 1st (= smallest) pivot, the second one encodes which elements are larger than the 1st and smaller than the
+ # 2nd pivot, ..., and the last encodes which elements are larger than last ( = largest) pivot.
+ # Now each processor has partitioned its locally sorted data into P partitions.
+ # To obtain the globally sorted result, each processor needs to send its i-th partition to processor i so that
+ # each processor i finally holds all samples from partition i.
+ # This all-to-all communication requires information about how many values will be sent for each partition.
+ # Processor i keeps i-th partitions and sends j-th partition to processor j.
+ local_partition_sizes = torch.sum(
+ lt_partitions, dim=1
+ ) # How many values are in each local partition?
+ print(
+ f"Rank {rank}/{size}: Local # elements to be sent to other ranks (keep own section): {local_partition_sizes}"
+ )
+ global_partition_sizes = torch.zeros_like(
+ local_partition_sizes
+ ) # How many values are in each global partition?
+ comm.Allreduce(local_partition_sizes, global_partition_sizes, op=MPI.SUM)
+ if rank == 0:
+ print(
+ f"Global # of elements on all ranks (partition matrix): {global_partition_sizes}"
+ )
+ # Construct matrix holding information which local value will be shipped where.
+ communication_index_vector = torch.empty_like(local_sorted, dtype=torch.int64)
+ # Loop over `lt_partitions` (binary encoding of which elements is in which partition formed by pivots).
+ for idx_rank, partition_mask_value in enumerate(lt_partitions):
+ communication_index_vector[partition_mask_value > 0] = idx_rank
+ # Elements in 0th partition (< first pivot) get 0, i.e., will be collected at rank 0, elements in 1st
+ # partition (> than first + < than second pivot) get 1, i.e., will be collected at rank 1,...
+ print(f"Rank {rank}/{size}: Ship element to rank: {communication_index_vector}")
+ # Counts and displacements for Alltoallv are rank-specific!
+ # send_counts_local on rank i: Integer array, entry j specifies number of values to be sent to rank j.
+ # recv_counts_local on rank i: Integer array, entry j specifies number of values to be received from rank j.
+ # Determine how many elements are to be shipped to which rank.
+ send_counts_local = numpy.zeros(size, dtype=int)
+ # Determine number of elements in each local partition.
+ for idx_rank in numpy.arange(size):
+ send_counts_local[idx_rank] = int((communication_index_vector == idx_rank).sum(dim=0))
+ # Determine local send displacements from local send counts.
+ send_displ_local = numpy.zeros(size, dtype=int)
+ send_displ_local[1:] = numpy.cumsum(send_counts_local, axis=0)[:-1]
+ # Allgather local send counts to determine how many elements are sent by other ranks to this rank and thus are
+ # to be received.
+ send_counts_global = numpy.zeros((size, size), dtype=int)
+ comm.Allgather([send_counts_local, MPI.INT], [send_counts_global, MPI.INT])
+ # Determine global receive counts from transposed global send counts.
+ recv_counts_global = numpy.transpose(send_counts_global)
+ # Extract local receive counts.
+ recv_counts_local = recv_counts_global[rank]
+ # Determine local receive displacements from local receive counts.
+ recv_displ_local = numpy.zeros(size, dtype=int)
+ recv_displ_local[1:] = numpy.cumsum(recv_counts_local, axis=0)[:-1]
+
+ value_buffer = torch.zeros((global_partition_sizes[rank],), dtype=local_sorted.dtype)
+ # Communicating torch tensors using buffered MPI variants requires accessing the tensor's storage pointer!
+ send_buf = [
+ MPI.memory.fromaddress(local_sorted.data_ptr(), 0),
+ (send_counts_local.tolist(), send_displ_local.tolist()),
+ __mpi_type_mappings[local_sorted.dtype],
+ ]
+ recv_buf = [
+ MPI.memory.fromaddress(value_buffer.data_ptr(), 0),
+ (recv_counts_local.tolist(), recv_displ_local.tolist()),
+ __mpi_type_mappings[value_buffer.dtype],
+ ]
+ comm.Alltoallv(send_buf, recv_buf)
+ result, _ = torch.sort(value_buffer)
+ return result
+
+
+if __name__ == "__main__":
+ data_path = "/pfs/work7/workspace/scratch/ku4408-VL-ScalableAI/data/psrs_data.h5"
+
+ parser = argparse.ArgumentParser(prog="Parallel Sorting by Regular Samples")
+ parser.add_argument(
+ "--dataset",
+ type=str,
+ default="duplicates_1",
+ help="The dataset to be sorted.",
+ )
+
+ args = parser.parse_args()
+
+ comm = MPI.COMM_WORLD
+ rank, size = comm.rank, comm.size
+
+ with h5py.File(data_path, "r") as f:
+ chunk = int(f[args.dataset].shape[0] / size)
+ if rank == size - 1:
+ data = torch.tensor(f[args.dataset][rank * chunk:])
+ else:
+ data = torch.tensor(f[args.dataset][rank * chunk:(rank + 1) * chunk])
+
+ if rank == 0:
+ print(
+ "########\n"
+ "# PSRS #\n"
+ "########"
+ )
+
+ print(f"Local data on rank {rank} = {data}")
+
+ if rank == 0:
+ print("Start sorting...")
+ start = time.perf_counter()
+ result = sort(data)
+ elapsed_local = time.perf_counter() - start
+ elapsed_global = comm.allreduce(elapsed_local, op=MPI.SUM)
+ elapsed_global /= size
+ if rank == 0:
+ print(f"Sorting done...\nRank-averaged run time: {elapsed_global} s")
+ print(f"Sorted chunk on rank {rank}/{size}: {result}")
diff --git a/2_psrs_cdist/solutions/A2/psrs_job.out b/2_psrs_cdist/solutions/A2/psrs_job.out
new file mode 100644
index 0000000..bb72d12
--- /dev/null
+++ b/2_psrs_cdist/solutions/A2/psrs_job.out
@@ -0,0 +1,324 @@
+NO DUPLICATES
+Local data on rank 1 = tensor([912731183, 238291756, 31221075, ..., 487613626, 531157405,
+ 418984112])
+Local data on rank 3 = tensor([293658335, 929892148, 683059664, ..., 149632860, 945975511,
+ 593779412])
+Local data on rank 2 = tensor([139483217, 885598716, 387917542, ..., 585719775, 959388810,
+ 228497015])
+########
+# PSRS #
+########
+Local data on rank 0 = tensor([318507780, 309697244, 42625426, ..., 962448579, 643638799,
+ 832257241])
+Start sorting...
+###########
+# PHASE 1 #
+###########
+Rank 2/4: Local sorting done...[OK]
+Rank 2/4: Number of elements in local chunk is 250000000.
+Rank 0/4: Local sorting done...[OK]
+Rank 0/4: Number of elements in local chunk is 250000000.
+Rank 3/4: Local sorting done...[OK]
+Rank 3/4: Number of elements in local chunk is 250000000.
+Rank 1/4: Local sorting done...[OK]
+Rank 1/4: Number of elements in local chunk is 250000000.
+Rank 2/4: There are 4 local regular samples: tensor([ 14, 250024707, 500024175, 750032344])
+Rank 1/4: There are 4 local regular samples: tensor([ 5, 250003006, 499999449, 749978522])
+Rank 0/4: There are 4 local regular samples: tensor([ 0, 249993236, 500002095, 750013289])
+Overall number of regular samples is 16.
+Rank 3/4: There are 4 local regular samples: tensor([ 3, 249978058, 499974006, 749975837])
+On root: Regular samples gathered...[OK]
+###########
+# PHASE 2 #
+###########
+On root: Regular samples are tensor([ 0, 3, 5, 14, 249978058, 249993236,
+ 250003006, 250024707, 499974006, 499999449, 500002095, 500024175,
+ 749975837, 749978522, 750013289, 750032344]).
+On root: There are 3 global pivots: tensor([249993236, 499999449, 749978522])
+Pivots broadcast to all processes...
+###########
+# PHASE 3 #
+###########
+Each processor forms P disjunct partitions of locally sorted elements using pivots as splits.
+Iterate over pivots to find index of first pivot > element's value.
+Rank 2/4: Local # elements to be sent to other ranks (keep own section): tensor([62491991, 62501751, 62492862, 62513396])
+Rank 1/4: Local # elements to be sent to other ranks (keep own section): tensor([62497531, 62502469, 62500000, 62500000])
+Rank 0/4: Local # elements to be sent to other ranks (keep own section): tensor([62500000, 62499299, 62491983, 62508718])
+Rank 3/4: Local # elements to be sent to other ranks (keep own section): tensor([62503714, 62502694, 62494228, 62499364])
+Global # of elements on all ranks (partition matrix): tensor([249993236, 250006213, 249979073, 250021478])
+Rank 2/4: Ship element to rank: tensor([0, 0, 0, ..., 3, 3, 3])
+Rank 1/4: Ship element to rank: tensor([0, 0, 0, ..., 3, 3, 3])
+Rank 3/4: Ship element to rank: tensor([0, 0, 0, ..., 3, 3, 3])
+Rank 0/4: Ship element to rank: tensor([0, 0, 0, ..., 3, 3, 3])
+Sorting done...
+Rank-averaged run time: 55.73376413062215 s
+Sorted chunk on rank 2/4: tensor([499999449, 499999450, 499999451, ..., 749978519, 749978520,
+ 749978521])
+Sorted chunk on rank 1/4: tensor([249993236, 249993237, 249993238, ..., 499999446, 499999447,
+ 499999448])
+Sorted chunk on rank 0/4: tensor([ 0, 1, 2, ..., 249993233, 249993234,
+ 249993235])
+Sorted chunk on rank 3/4: tensor([749978522, 749978523, 749978524, ..., 999999997, 999999998,
+ 999999999])
+5 % DUPLICATES
+Local data on rank 2 = tensor([477468733, 127733109, 627602415, ..., 855353275, 26155210,
+ 576852549])
+########
+# PSRS #
+########
+Local data on rank 0 = tensor([355926583, 101881567, 667655099, ..., 372829943, 905728480,
+ 453198313])
+Start sorting...
+###########
+# PHASE 1 #
+###########
+Local data on rank 3 = tensor([822153593, 69920072, 291463329, ..., 511904625, 876475388,
+ 127929309])
+Local data on rank 1 = tensor([435319384, 474554791, 761429556, ..., 678039451, 869716054,
+ 477803583])
+Rank 2/4: Local sorting done...[OK]
+Rank 2/4: Number of elements in local chunk is 250000000.
+Rank 0/4: Local sorting done...[OK]
+Rank 0/4: Number of elements in local chunk is 250000000.
+Rank 3/4: Local sorting done...[OK]
+Rank 3/4: Number of elements in local chunk is 250000000.
+Rank 1/4: Local sorting done...[OK]
+Rank 1/4: Number of elements in local chunk is 250000000.
+Rank 2/4: There are 4 local regular samples: tensor([ 4, 200020790, 450036864, 699967569])
+Rank 1/4: There are 4 local regular samples: tensor([ 0, 199968853, 449934061, 700004704])
+Rank 0/4: There are 4 local regular samples: tensor([ 0, 200004021, 449997671, 700000078])
+Overall number of regular samples is 16.
+Rank 3/4: There are 4 local regular samples: tensor([ 3, 200006078, 450030998, 700027765])
+On root: Regular samples gathered...[OK]
+###########
+# PHASE 2 #
+###########
+On root: Regular samples are tensor([ 0, 0, 3, 4, 199968853, 200004021,
+ 200006078, 200020790, 449934061, 449997671, 450030998, 450036864,
+ 699967569, 700000078, 700004704, 700027765]).
+On root: There are 3 global pivots: tensor([200004021, 449997671, 700000078])
+Pivots broadcast to all processes...
+###########
+# PHASE 3 #
+###########
+Each processor forms P disjunct partitions of locally sorted elements using pivots as splits.
+Iterate over pivots to find index of first pivot > element's value.
+Rank 2/4: Local # elements to be sent to other ranks (keep own section): tensor([62495807, 62494516, 62517739, 62491938])
+Rank 1/4: Local # elements to be sent to other ranks (keep own section): tensor([62508697, 62507129, 62483019, 62501155])
+Rank 0/4: Local # elements to be sent to other ranks (keep own section): tensor([62500000, 62500000, 62500000, 62500000])
+Rank 3/4: Local # elements to be sent to other ranks (keep own section): tensor([62499517, 62492005, 62501649, 62506829])
+Global # of elements on all ranks (partition matrix): tensor([250004021, 249993650, 250002407, 249999922])
+Rank 2/4: Ship element to rank: tensor([0, 0, 0, ..., 3, 3, 3])
+Rank 1/4: Ship element to rank: tensor([0, 0, 0, ..., 3, 3, 3])
+Rank 0/4: Ship element to rank: tensor([0, 0, 0, ..., 3, 3, 3])
+Rank 3/4: Ship element to rank: tensor([0, 0, 0, ..., 3, 3, 3])
+Sorting done...
+Rank-averaged run time: 54.29611164028756 s
+Sorted chunk on rank 2/4: tensor([449997671, 449997672, 449997673, ..., 700000075, 700000076,
+ 700000077])
+Sorted chunk on rank 1/4: tensor([200004021, 200004022, 200004023, ..., 449997668, 449997669,
+ 449997670])
+Sorted chunk on rank 3/4: tensor([700000078, 700000079, 700000080, ..., 949999997, 949999998,
+ 949999999])
+Sorted chunk on rank 0/4: tensor([ 0, 0, 1, ..., 200004018, 200004019,
+ 200004020])
+10 % DUPLICATES
+########
+# PSRS #
+########
+Local data on rank 0 = tensor([305642785, 286323505, 728015630, ..., 682317156, 798373445,
+ 895146073])
+Start sorting...
+###########
+# PHASE 1 #
+###########
+Local data on rank 3 = tensor([455175700, 393254749, 179903063, ..., 436593859, 623044534,
+ 306300192])
+Local data on rank 1 = tensor([398018127, 21608708, 22914288, ..., 34843052, 120412798,
+ 57185758])
+Local data on rank 2 = tensor([629870937, 336179000, 75229887, ..., 708722263, 297832523,
+ 782428402])
+Rank 2/4: Local sorting done...[OK]
+Rank 2/4: Number of elements in local chunk is 250000000.
+Rank 0/4: Local sorting done...[OK]
+Rank 0/4: Number of elements in local chunk is 250000000.
+Rank 3/4: Local sorting done...[OK]
+Rank 3/4: Number of elements in local chunk is 250000000.
+Rank 1/4: Local sorting done...[OK]
+Rank 1/4: Number of elements in local chunk is 250000000.
+Rank 2/4: There are 4 local regular samples: tensor([ 0, 149960401, 399957493, 649941868])
+Rank 0/4: There are 4 local regular samples: tensor([ 2, 149996075, 399976515, 649983460])
+Overall number of regular samples is 16.
+Rank 3/4: There are 4 local regular samples: tensor([ 2, 150043131, 400077060, 650034148])
+Rank 1/4: There are 4 local regular samples: tensor([ 0, 150000777, 399988417, 650040483])
+On root: Regular samples gathered...[OK]
+###########
+# PHASE 2 #
+###########
+On root: Regular samples are tensor([ 0, 0, 2, 2, 149960401, 149996075,
+ 150000777, 150043131, 399957493, 399976515, 399988417, 400077060,
+ 649941868, 649983460, 650034148, 650040483]).
+On root: There are 3 global pivots: tensor([149996075, 399976515, 649983460])
+Pivots broadcast to all processes...
+###########
+# PHASE 3 #
+###########
+Each processor forms P disjunct partitions of locally sorted elements using pivots as splits.
+Iterate over pivots to find index of first pivot > element's value.
+Rank 2/4: Local # elements to be sent to other ranks (keep own section): tensor([62509008, 62495689, 62505639, 62489664])
+Rank 1/4: Local # elements to be sent to other ranks (keep own section): tensor([62498814, 62498275, 62488634, 62514277])
+Rank 0/4: Local # elements to be sent to other ranks (keep own section): tensor([62500000, 62500000, 62500000, 62500000])
+Rank 3/4: Local # elements to be sent to other ranks (keep own section): tensor([62488253, 62486476, 62512672, 62512599])
+Global # of elements on all ranks (partition matrix): tensor([249996075, 249980440, 250006945, 250016540])
+Rank 1/4: Ship element to rank: tensor([0, 0, 0, ..., 3, 3, 3])
+Rank 3/4: Ship element to rank: tensor([0, 0, 0, ..., 3, 3, 3])
+Rank 0/4: Ship element to rank: tensor([0, 0, 0, ..., 3, 3, 3])
+Rank 2/4: Ship element to rank: tensor([0, 0, 0, ..., 3, 3, 3])
+Sorting done...
+Rank-averaged run time: 56.31642022822052 s
+Sorted chunk on rank 2/4: tensor([399976515, 399976516, 399976517, ..., 649983457, 649983458,
+ 649983459])
+Sorted chunk on rank 1/4: tensor([149996075, 149996076, 149996077, ..., 399976512, 399976513,
+ 399976514])
+Sorted chunk on rank 3/4: tensor([649983460, 649983461, 649983462, ..., 899999997, 899999998,
+ 899999999])
+Sorted chunk on rank 0/4: tensor([ 0, 0, 1, ..., 149996072, 149996073,
+ 149996074])
+UNIFORMLY DISTRIBUTED RANDOM NUMBERS
+Local data on rank 3 = tensor([709242213, 64999136, 764543656, ..., 494043760, 642395880,
+ 59439745])
+Local data on rank 2 = tensor([ 38492987, 899451823, 627017129, ..., 631853279, 807322227,
+ 506987459])
+Local data on rank 1 = tensor([313485550, 566343157, 585037407, ..., 348865800, 438628716,
+ 166575358])
+########
+# PSRS #
+########
+Local data on rank 0 = tensor([922253299, 694846461, 538350466, ..., 461620389, 673297837,
+ 754540488])
+Start sorting...
+###########
+# PHASE 1 #
+###########
+Rank 2/4: Local sorting done...[OK]
+Rank 2/4: Number of elements in local chunk is 250000000.
+Rank 3/4: Local sorting done...[OK]
+Rank 3/4: Number of elements in local chunk is 250000000.
+Rank 0/4: Local sorting done...[OK]
+Rank 0/4: Number of elements in local chunk is 250000000.
+Rank 1/4: Local sorting done...[OK]
+Rank 1/4: Number of elements in local chunk is 250000000.
+Rank 2/4: There are 4 local regular samples: tensor([ 2, 249976531, 499962443, 749982363])
+Rank 3/4: There are 4 local regular samples: tensor([ 13, 250010478, 499987015, 749991792])
+Rank 0/4: There are 4 local regular samples: tensor([ 4, 250044290, 500012799, 750017480])
+Overall number of regular samples is 16.
+Rank 1/4: There are 4 local regular samples: tensor([ 8, 250002905, 499985043, 749948844])
+On root: Regular samples gathered...[OK]
+###########
+# PHASE 2 #
+###########
+On root: Regular samples are tensor([ 2, 4, 8, 13, 249976531, 250002905,
+ 250010478, 250044290, 499962443, 499985043, 499987015, 500012799,
+ 749948844, 749982363, 749991792, 750017480]).
+On root: There are 3 global pivots: tensor([250002905, 499985043, 749982363])
+Pivots broadcast to all processes...
+###########
+# PHASE 3 #
+###########
+Each processor forms P disjunct partitions of locally sorted elements using pivots as splits.
+Iterate over pivots to find index of first pivot > element's value.
+Rank 2/4: Local # elements to be sent to other ranks (keep own section): tensor([62506712, 62498904, 62494384, 62500000])
+Rank 1/4: Local # elements to be sent to other ranks (keep own section): tensor([62500000, 62500000, 62508377, 62491623])
+Rank 0/4: Local # elements to be sent to other ranks (keep own section): tensor([62489604, 62503304, 62498354, 62508738])
+Rank 3/4: Local # elements to be sent to other ranks (keep own section): tensor([62498087, 62501420, 62498169, 62502324])
+Global # of elements on all ranks (partition matrix): tensor([249994403, 250003628, 249999284, 250002685])
+Rank 2/4: Ship element to rank: tensor([0, 0, 0, ..., 3, 3, 3])
+Rank 0/4: Ship element to rank: tensor([0, 0, 0, ..., 3, 3, 3])
+Rank 3/4: Ship element to rank: tensor([0, 0, 0, ..., 3, 3, 3])
+Rank 1/4: Ship element to rank: tensor([0, 0, 0, ..., 3, 3, 3])
+Sorting done...
+Rank-averaged run time: 48.48549936176278 s
+Sorted chunk on rank 2/4: tensor([499985043, 499985043, 499985044, ..., 749982360, 749982361,
+ 749982362])
+Sorted chunk on rank 0/4: tensor([ 2, 4, 6, ..., 250002903, 250002904,
+ 250002904])
+Sorted chunk on rank 1/4: tensor([250002905, 250002907, 250002908, ..., 499985040, 499985040,
+ 499985040])
+Sorted chunk on rank 3/4: tensor([749982363, 749982363, 749982363, ..., 999999997, 999999997,
+ 999999999])
+TRIPLICATES
+Local data on rank 2 = tensor([ 79539910, 62860751, 38149835, ..., 66484620, 61227715,
+ 281163448])
+Local data on rank 3 = tensor([121669805, 404791105, 134876963, ..., 104417323, 226906130,
+ 837291])
+########
+# PSRS #
+########
+Local data on rank 0 = tensor([269955182, 112494896, 223541994, ..., 8608940, 119366620,
+ 106471758])
+Start sorting...
+###########
+# PHASE 1 #
+###########
+Local data on rank 1 = tensor([132545061, 314851768, 105793976, ..., 289208082, 188654007,
+ 109028272])
+Rank 2/4: Local sorting done...[OK]
+Rank 2/4: Number of elements in local chunk is 250000000.
+Rank 0/4: Local sorting done...[OK]
+Rank 0/4: Number of elements in local chunk is 250000000.
+Rank 3/4: Local sorting done...[OK]
+Rank 3/4: Number of elements in local chunk is 250000000.
+Rank 1/4: Local sorting done...[OK]
+Rank 1/4: Number of elements in local chunk is 250000000.
+Rank 2/4: There are 4 local regular samples: tensor([ 0, 83326870, 166667137, 249993573])
+Rank 0/4: There are 4 local regular samples: tensor([ 0, 83337833, 166666474, 250034636])
+Overall number of regular samples is 16.
+Rank 1/4: There are 4 local regular samples: tensor([ 0, 83340613, 166674265, 249999232])
+Rank 3/4: There are 4 local regular samples: tensor([ 1, 83327965, 166658906, 249995719])
+On root: Regular samples gathered...[OK]
+###########
+# PHASE 2 #
+###########
+On root: Regular samples are tensor([ 0, 0, 0, 1, 83326870, 83327965,
+ 83337833, 83340613, 166658906, 166666474, 166667137, 166674265,
+ 249993573, 249995719, 249999232, 250034636]).
+On root: There are 3 global pivots: tensor([ 83327965, 166666474, 249995719])
+Pivots broadcast to all processes...
+###########
+# PHASE 3 #
+###########
+Each processor forms P disjunct partitions of locally sorted elements using pivots as splits.
+Iterate over pivots to find index of first pivot > element's value.
+Rank 2/4: Local # elements to be sent to other ranks (keep own section): tensor([62500862, 62498626, 62502178, 62498334])
+Rank 1/4: Local # elements to be sent to other ranks (keep own section): tensor([62490465, 62503751, 62503140, 62502644])
+Rank 0/4: Local # elements to be sent to other ranks (keep own section): tensor([62492568, 62507432, 62488136, 62511864])
+Rank 3/4: Local # elements to be sent to other ranks (keep own section): tensor([62500000, 62505718, 62494281, 62500001])
+Global # of elements on all ranks (partition matrix): tensor([249983895, 250015527, 249987735, 250012843])
+Rank 2/4: Ship element to rank: tensor([0, 0, 0, ..., 3, 3, 3])
+Rank 1/4: Ship element to rank: tensor([0, 0, 0, ..., 3, 3, 3])
+Rank 3/4: Ship element to rank: tensor([0, 0, 0, ..., 3, 3, 3])
+Rank 0/4: Ship element to rank: tensor([0, 0, 0, ..., 3, 3, 3])
+Sorting done...
+Rank-averaged run time: 56.163604924455285 s
+Sorted chunk on rank 2/4: tensor([166666474, 166666474, 166666474, ..., 249995718, 249995718,
+ 249995718])
+Sorted chunk on rank 3/4: tensor([249995719, 249995719, 249995719, ..., 499999997, 499999998,
+ 499999999])
+Sorted chunk on rank 0/4: tensor([ 0, 0, 0, ..., 83327964, 83327964, 83327964])
+Sorted chunk on rank 1/4: tensor([ 83327965, 83327965, 83327965, ..., 166666473, 166666473,
+ 166666473])
+
+============================= JOB FEEDBACK =============================
+
+NodeName=uc2n[001-004]
+Job ID: 24894350
+Cluster: uc2
+User/Group: ku4408/scc
+State: COMPLETED (exit code 0)
+Nodes: 4
+Cores per node: 80
+CPU Utilized: 00:20:19
+CPU Efficiency: 1.00% of 1-09:46:40 core-walltime
+Job Wall-clock time: 00:06:20
+Memory Utilized: 76.59 GB (estimated maximum)
+Memory Efficiency: 21.79% of 351.56 GB (1.10 GB/core)
--
GitLab