A Discrete-Event Network Simulator
API
simple-distributed-mpi-comm.cc
Go to the documentation of this file.
1 /*
2  * Copyright 2018. Lawrence Livermore National Security, LLC.
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License version 2 as
6  * published by the Free Software Foundation;
7  *
8  * This program is distributed in the hope that it will be useful,
9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11  * GNU General Public License for more details.
12  *
13  * You should have received a copy of the GNU General Public License
14  * along with this program; if not, write to the Free Software
15  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16  *
17  * Author: Steven Smith <smith84@llnl.gov>
18  */
19 
57 #include "mpi-test-fixtures.h"
58 
59 #include "ns3/core-module.h"
60 #include "ns3/internet-stack-helper.h"
61 #include "ns3/ipv4-address-helper.h"
62 #include "ns3/ipv4-global-routing-helper.h"
63 #include "ns3/ipv4-list-routing-helper.h"
64 #include "ns3/ipv4-static-routing-helper.h"
65 #include "ns3/mpi-interface.h"
66 #include "ns3/network-module.h"
67 #include "ns3/nix-vector-helper.h"
68 #include "ns3/on-off-helper.h"
69 #include "ns3/packet-sink-helper.h"
70 #include "ns3/packet-sink.h"
71 #include "ns3/point-to-point-helper.h"
72 
73 #include <mpi.h>
74 
75 using namespace ns3;
76 
77 NS_LOG_COMPONENT_DEFINE("SimpleDistributedMpiComm");
78 
84 const int NS_COLOR = 1;
85 const int NOT_NS_COLOR = NS_COLOR + 1;
86 
95 void
96 ReportRank(int color, MPI_Comm splitComm)
97 {
98  int otherId = 0;
99  int otherSize = 1;
100 
101  MPI_Comm_rank(splitComm, &otherId);
102  MPI_Comm_size(splitComm, &otherSize);
103 
104  if (color == NS_COLOR)
105  {
106  RANK0COUT("ns-3 rank: ");
107  }
108  else
109  {
110  RANK0COUT("Other rank: ");
111  }
112 
113  RANK0COUTAPPEND("in MPI_COMM_WORLD: " << SinkTracer::GetWorldRank() << ":"
114  << SinkTracer::GetWorldSize() << ", in splitComm: "
115  << otherId << ":" << otherSize << std::endl);
116 
117 } // ReportRank()
118 
119 int
120 main(int argc, char* argv[])
121 {
122  bool nix = true;
123  bool nullmsg = false;
124  bool tracing = false;
125  bool init = false;
126  bool verbose = false;
127  bool testing = false;
128 
129  // Parse command line
130  CommandLine cmd(__FILE__);
131  cmd.AddValue("nix", "Enable the use of nix-vector or global routing", nix);
132  cmd.AddValue("nullmsg",
133  "Enable the use of null-message synchronization (instead of granted time window)",
134  nullmsg);
135  cmd.AddValue("tracing", "Enable pcap tracing", tracing);
136  cmd.AddValue("init", "ns-3 should initialize MPI by calling MPI_Init", init);
137  cmd.AddValue("verbose", "verbose output", verbose);
138  cmd.AddValue("test", "Enable regression test output", testing);
139  cmd.Parse(argc, argv);
140 
141  // Defer reporting the configuration until we know the communicator
142 
143  // Distributed simulation setup; by default use granted time window algorithm.
144  if (nullmsg)
145  {
146  GlobalValue::Bind("SimulatorImplementationType",
147  StringValue("ns3::NullMessageSimulatorImpl"));
148  }
149  else
150  {
151  GlobalValue::Bind("SimulatorImplementationType",
152  StringValue("ns3::DistributedSimulatorImpl"));
153  }
154 
155  // MPI_Init
156 
157  if (init)
158  {
159  // Initialize MPI directly
160  MPI_Init(&argc, &argv);
161  }
162  else
163  {
164  // Let ns-3 call MPI_Init and MPI_Finalize
165  MpiInterface::Enable(&argc, &argv);
166  }
167 
169 
170  auto worldSize = SinkTracer::GetWorldSize();
171  auto worldRank = SinkTracer::GetWorldRank();
172 
173  if ((!init) && (worldSize != 2))
174  {
175  RANK0COUT("This simulation requires exactly 2 logical processors if --init is not set."
176  << std::endl);
177  return 1;
178  }
179 
180  if (worldSize < 2)
181  {
182  RANK0COUT("This simulation requires 2 or more logical processors." << std::endl);
183  return 1;
184  }
185 
186  // Set up the MPI communicator for ns-3
187  // Condition ns-3 Communicator
188  // a. worldSize = 2 copy of MPI_COMM_WORLD
189  // b. worldSize > 2 communicator of ranks 1-2
190 
191  // Flag to record that we created a communicator so we can free it at the end.
192  bool freeComm = false;
193  // The new communicator, if we create one
194  MPI_Comm splitComm = MPI_COMM_WORLD;
195  // The list of ranks assigned to ns-3
196  std::string ns3Ranks;
197  // Tag for whether this rank should go into a new communicator
198  int color = MPI_UNDEFINED;
199 
200  if (worldSize == 2)
201  {
202  std::stringstream ss;
203  color = NS_COLOR;
204  ss << "MPI_COMM_WORLD (" << worldSize << " ranks)";
205  ns3Ranks = ss.str();
206  splitComm = MPI_COMM_WORLD;
207  freeComm = false;
208  }
209  else
210  {
211  // worldSize > 2 communicator of ranks 1-2
212 
213  // Put ranks 1-2 in the new communicator
214  if (worldRank == 1 || worldRank == 2)
215  {
216  color = NS_COLOR;
217  }
218  else
219  {
220  color = NOT_NS_COLOR;
221  }
222  std::stringstream ss;
223  ss << "Split [1-2] (out of " << worldSize << " ranks) from MPI_COMM_WORLD";
224  ns3Ranks = ss.str();
225 
226  // Now create the new communicator
227  MPI_Comm_split(MPI_COMM_WORLD, color, worldRank, &splitComm);
228  freeComm = true;
229  }
230 
231  if (init)
232  {
233  MpiInterface::Enable(splitComm);
234  }
235 
236  // Report the configuration from rank 0 only
237  RANK0COUT(cmd.GetName() << "\n");
238  RANK0COUT("\n");
239  RANK0COUT("Configuration:\n");
240  RANK0COUT("Routing: " << (nix ? "nix-vector" : "global") << "\n");
241  RANK0COUT("Synchronization: " << (nullmsg ? "null-message" : "granted time window (YAWNS)")
242  << "\n");
243  RANK0COUT("MPI_Init called: "
244  << (init ? "explicitly by this program" : "implicitly by ns3::MpiInterface::Enable()")
245  << "\n");
246  RANK0COUT("ns-3 Communicator: " << ns3Ranks << "\n");
247  RANK0COUT("PCAP tracing: " << (tracing ? "" : "not") << " enabled\n");
248  RANK0COUT("\n");
249  RANK0COUT("Rank assignments:" << std::endl);
250 
251  if (worldRank == 0)
252  {
253  ReportRank(color, splitComm);
254  }
255 
256  if (verbose)
257  {
258  // Circulate a token to have each rank report in turn
259  int token;
260 
261  if (worldRank == 0)
262  {
263  token = 1;
264  }
265  else
266  {
267  MPI_Recv(&token, 1, MPI_INT, worldRank - 1, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
268  ReportRank(color, splitComm);
269  }
270 
271  MPI_Send(&token, 1, MPI_INT, (worldRank + 1) % worldSize, 0, MPI_COMM_WORLD);
272 
273  if (worldRank == 0)
274  {
275  MPI_Recv(&token, 1, MPI_INT, worldSize - 1, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
276  }
277  } // circulate token to report rank
278 
279  RANK0COUT(std::endl);
280 
281  if (color != NS_COLOR)
282  {
283  // Do other work outside the ns-3 communicator
284 
285  // In real use of a separate communicator from ns-3
286  // the other tasks would be running another simulator
287  // or other desired work here..
288 
289  // Our work is done, just wait for everyone else to finish.
290 
292 
293  if (init)
294  {
295  MPI_Finalize();
296  }
297 
298  return 0;
299  }
300 
301  // The code below here is essentially the same as simple-distributed.cc
302  // --------------------------------------------------------------------
303 
304  // We use a trace instead of relying on NS_LOG
305 
306  if (verbose)
307  {
308  LogComponentEnable("PacketSink", LOG_LEVEL_INFO);
309  }
310 
311  uint32_t systemId = MpiInterface::GetSystemId();
312  uint32_t systemCount = MpiInterface::GetSize();
313 
314  // Check for valid distributed parameters.
315  // Both this script and simple-distributed.cc will work
316  // with arbitrary numbers of ranks, as long as there are at least 2.
317  if (systemCount < 2)
318  {
319  RANK0COUT("This simulation requires at least 2 logical processors." << std::endl);
320  return 1;
321  }
322 
323  // Some default values
324  Config::SetDefault("ns3::OnOffApplication::PacketSize", UintegerValue(512));
325  Config::SetDefault("ns3::OnOffApplication::DataRate", StringValue("1Mbps"));
326  Config::SetDefault("ns3::OnOffApplication::MaxBytes", UintegerValue(512));
327 
328  // Create leaf nodes on left with system id 0
329  NodeContainer leftLeafNodes;
330  leftLeafNodes.Create(4, 0);
331 
332  // Create router nodes. Left router
333  // with system id 0, right router with
334  // system id 1
335  NodeContainer routerNodes;
336  Ptr<Node> routerNode1 = CreateObject<Node>(0);
337  Ptr<Node> routerNode2 = CreateObject<Node>(1);
338  routerNodes.Add(routerNode1);
339  routerNodes.Add(routerNode2);
340 
341  // Create leaf nodes on left with system id 1
342  NodeContainer rightLeafNodes;
343  rightLeafNodes.Create(4, 1);
344 
345  PointToPointHelper routerLink;
346  routerLink.SetDeviceAttribute("DataRate", StringValue("5Mbps"));
347  routerLink.SetChannelAttribute("Delay", StringValue("5ms"));
348 
349  PointToPointHelper leafLink;
350  leafLink.SetDeviceAttribute("DataRate", StringValue("1Mbps"));
351  leafLink.SetChannelAttribute("Delay", StringValue("2ms"));
352 
353  // Add link connecting routers
354  NetDeviceContainer routerDevices;
355  routerDevices = routerLink.Install(routerNodes);
356 
357  // Add links for left side leaf nodes to left router
358  NetDeviceContainer leftRouterDevices;
359  NetDeviceContainer leftLeafDevices;
360  for (uint32_t i = 0; i < 4; ++i)
361  {
362  NetDeviceContainer temp = leafLink.Install(leftLeafNodes.Get(i), routerNodes.Get(0));
363  leftLeafDevices.Add(temp.Get(0));
364  leftRouterDevices.Add(temp.Get(1));
365  }
366 
367  // Add links for right side leaf nodes to right router
368  NetDeviceContainer rightRouterDevices;
369  NetDeviceContainer rightLeafDevices;
370  for (uint32_t i = 0; i < 4; ++i)
371  {
372  NetDeviceContainer temp = leafLink.Install(rightLeafNodes.Get(i), routerNodes.Get(1));
373  rightLeafDevices.Add(temp.Get(0));
374  rightRouterDevices.Add(temp.Get(1));
375  }
376 
379  Ipv4StaticRoutingHelper staticRouting;
380 
382  list.Add(staticRouting, 0);
383  list.Add(nixRouting, 10);
384 
385  if (nix)
386  {
387  stack.SetRoutingHelper(list); // has effect on the next Install ()
388  }
389 
390  stack.InstallAll();
391 
392  Ipv4InterfaceContainer routerInterfaces;
393  Ipv4InterfaceContainer leftLeafInterfaces;
394  Ipv4InterfaceContainer leftRouterInterfaces;
395  Ipv4InterfaceContainer rightLeafInterfaces;
396  Ipv4InterfaceContainer rightRouterInterfaces;
397 
398  Ipv4AddressHelper leftAddress;
399  leftAddress.SetBase("10.1.1.0", "255.255.255.0");
400 
401  Ipv4AddressHelper routerAddress;
402  routerAddress.SetBase("10.2.1.0", "255.255.255.0");
403 
404  Ipv4AddressHelper rightAddress;
405  rightAddress.SetBase("10.3.1.0", "255.255.255.0");
406 
407  // Router-to-Router interfaces
408  routerInterfaces = routerAddress.Assign(routerDevices);
409 
410  // Left interfaces
411  for (uint32_t i = 0; i < 4; ++i)
412  {
413  NetDeviceContainer ndc;
414  ndc.Add(leftLeafDevices.Get(i));
415  ndc.Add(leftRouterDevices.Get(i));
416  Ipv4InterfaceContainer ifc = leftAddress.Assign(ndc);
417  leftLeafInterfaces.Add(ifc.Get(0));
418  leftRouterInterfaces.Add(ifc.Get(1));
419  leftAddress.NewNetwork();
420  }
421 
422  // Right interfaces
423  for (uint32_t i = 0; i < 4; ++i)
424  {
425  NetDeviceContainer ndc;
426  ndc.Add(rightLeafDevices.Get(i));
427  ndc.Add(rightRouterDevices.Get(i));
428  Ipv4InterfaceContainer ifc = rightAddress.Assign(ndc);
429  rightLeafInterfaces.Add(ifc.Get(0));
430  rightRouterInterfaces.Add(ifc.Get(1));
431  rightAddress.NewNetwork();
432  }
433 
434  if (!nix)
435  {
437  }
438 
439  if (tracing)
440  {
441  if (systemId == 0)
442  {
443  routerLink.EnablePcap("router-left", routerDevices, true);
444  leafLink.EnablePcap("leaf-left", leftLeafDevices, true);
445  }
446 
447  if (systemId == 1)
448  {
449  routerLink.EnablePcap("router-right", routerDevices, true);
450  leafLink.EnablePcap("leaf-right", rightLeafDevices, true);
451  }
452  }
453 
454  // Create a packet sink on the right leafs to receive packets from left leafs
455  uint16_t port = 50000;
456  if (systemId == 1)
457  {
458  Address sinkLocalAddress(InetSocketAddress(Ipv4Address::GetAny(), port));
459  PacketSinkHelper sinkHelper("ns3::UdpSocketFactory", sinkLocalAddress);
460  ApplicationContainer sinkApp;
461  for (uint32_t i = 0; i < 4; ++i)
462  {
463  auto apps = sinkHelper.Install(rightLeafNodes.Get(i));
464  auto sink = DynamicCast<PacketSink>(apps.Get(0));
465  NS_ASSERT_MSG(sink, "Couldn't get PacketSink application.");
466  if (testing)
467  {
468  sink->TraceConnectWithoutContext("RxWithAddresses",
470  }
471  sinkApp.Add(apps);
472  }
473  sinkApp.Start(Seconds(1.0));
474  sinkApp.Stop(Seconds(5));
475  }
476 
477  // Create the OnOff applications to send
478  if (systemId == 0)
479  {
480  OnOffHelper clientHelper("ns3::UdpSocketFactory", Address());
481  clientHelper.SetAttribute("OnTime", StringValue("ns3::ConstantRandomVariable[Constant=1]"));
482  clientHelper.SetAttribute("OffTime",
483  StringValue("ns3::ConstantRandomVariable[Constant=0]"));
484 
486  for (uint32_t i = 0; i < 4; ++i)
487  {
488  AddressValue remoteAddress(InetSocketAddress(rightLeafInterfaces.GetAddress(i), port));
489  clientHelper.SetAttribute("Remote", remoteAddress);
490  clientApps.Add(clientHelper.Install(leftLeafNodes.Get(i)));
491  }
492  clientApps.Start(Seconds(1.0));
493  clientApps.Stop(Seconds(5));
494  }
495 
496  RANK0COUT(std::endl);
497 
499  Simulator::Run();
501 
502  // --------------------------------------------------------------------
503  // Conditional cleanup based on whether we built a communicator
504  // and called MPI_Init
505 
506  if (freeComm)
507  {
508  MPI_Comm_free(&splitComm);
509  }
510 
511  if (testing)
512  {
514  }
515 
516  // Clean up the ns-3 MPI execution environment
517  // This will call MPI_Finalize if MpiInterface::Initialize was called
519 
520  if (init)
521  {
522  // We called MPI_Init, so we have to call MPI_Finalize
523  MPI_Finalize();
524  }
525 
526  return 0;
527 }
a polymophic address class
Definition: address.h:101
holds a vector of ns3::Application pointers.
void Start(Time start) const
Start all of the Applications in this container at the start time given as a parameter.
void Stop(Time stop) const
Arrange for all of the Applications in this container to Stop() at the Time given as a parameter.
void Add(ApplicationContainer other)
Append the contents of another ApplicationContainer to the end of this container.
Parse command-line arguments.
Definition: command-line.h:232
static void Bind(std::string name, const AttributeValue &value)
Iterate over the set of GlobalValues until a matching name is found and then set its value with Globa...
an Inet address class
aggregate IP/TCP/UDP functionality to existing Nodes.
A helper class to make life easier while doing simple IPv4 address assignment in scripts.
void SetBase(Ipv4Address network, Ipv4Mask mask, Ipv4Address base="0.0.0.1")
Set the base network number, network mask and base address.
Ipv4Address NewNetwork()
Increment the network number and reset the IP address counter to the base value provided in the SetBa...
Ipv4InterfaceContainer Assign(const NetDeviceContainer &c)
Assign IP addresses to the net devices specified in the container based on the current network prefix...
static Ipv4Address GetAny()
static void PopulateRoutingTables()
Build a routing database and initialize the routing tables of the nodes in the simulation.
holds a vector of std::pair of Ptr<Ipv4> and interface index.
std::pair< Ptr< Ipv4 >, uint32_t > Get(uint32_t i) const
Get the std::pair of an Ptr<Ipv4> and interface stored at the location specified by the index.
void Add(const Ipv4InterfaceContainer &other)
Concatenate the entries in the other container with ours.
Ipv4Address GetAddress(uint32_t i, uint32_t j=0) const
Helper class that adds ns3::Ipv4ListRouting objects.
Helper class that adds ns3::Ipv4StaticRouting objects.
static uint32_t GetSystemId()
Get the id number of this rank.
static uint32_t GetSize()
Get the number of ranks used by ns-3.
static void Disable()
Clean up the ns-3 parallel communications interface.
static void Enable(int *pargc, char ***pargv)
Setup the parallel communication interface.
holds a vector of ns3::NetDevice pointers
void Add(NetDeviceContainer other)
Append the contents of another NetDeviceContainer to the end of this container.
Ptr< NetDevice > Get(uint32_t i) const
Get the Ptr<NetDevice> stored in this container at a given index.
Helper class that adds Nix-vector routing to nodes.
keep track of a set of node pointers.
void Create(uint32_t n)
Create n nodes and append pointers to them to the end of this NodeContainer.
void Add(const NodeContainer &nc)
Append the contents of another NodeContainer to the end of this container.
Ptr< Node > Get(uint32_t i) const
Get the Ptr<Node> stored in this container at a given index.
bool TraceConnectWithoutContext(std::string name, const CallbackBase &cb)
Connect a TraceSource to a Callback without a context.
Definition: object-base.cc:315
A helper to make it easier to instantiate an ns3::OnOffApplication on a set of nodes.
Definition: on-off-helper.h:44
A helper to make it easier to instantiate an ns3::PacketSinkApplication on a set of nodes.
void EnablePcap(std::string prefix, Ptr< NetDevice > nd, bool promiscuous=false, bool explicitFilename=false)
Enable pcap output the indicated net device.
Build a set of PointToPointNetDevice objects.
void SetDeviceAttribute(std::string name, const AttributeValue &value)
Set an attribute value to be propagated to each NetDevice created by the helper.
void SetChannelAttribute(std::string name, const AttributeValue &value)
Set an attribute value to be propagated to each Channel created by the helper.
NetDeviceContainer Install(NodeContainer c)
static void Destroy()
Execute the events scheduled with ScheduleDestroy().
Definition: simulator.cc:142
static void Run()
Run the simulation.
Definition: simulator.cc:178
static void Stop()
Tell the Simulator the calling event should be the last one executed.
Definition: simulator.cc:186
static void SinkTrace(const ns3::Ptr< const ns3::Packet > packet, const ns3::Address &srcAddress, const ns3::Address &destAddress)
PacketSink receive trace callback.
static void Verify(unsigned long expectedCount)
Verify the sink trace count observed matches the expected count.
static void Init()
PacketSink Init.
static int GetWorldSize()
Get the MPI size of the world communicator.
static int GetWorldRank()
Get the MPI rank in the world communicator.
Hold variables of type string.
Definition: string.h:56
Hold an unsigned integer type.
Definition: uinteger.h:45
uint16_t port
Definition: dsdv-manet.cc:44
#define NS_ASSERT_MSG(condition, message)
At runtime, in debugging builds, if this condition is not true, the program prints the message to out...
Definition: assert.h:86
void SetDefault(std::string name, const AttributeValue &value)
Definition: config.cc:890
#define NS_LOG_COMPONENT_DEFINE(name)
Define a Log component with a specific name.
Definition: log.h:202
#define RANK0COUT(x)
Write to std::cout only from rank 0.
#define RANK0COUTAPPEND(x)
Append to std::cout only from rank 0.
Time Seconds(double value)
Construct a Time in the indicated unit.
Definition: nstime.h:1326
Common methods for MPI examples.
clientApps
Definition: first.py:64
stack
Definition: first.py:44
Every class exported by the ns3 library is enclosed in the ns3 namespace.
void LogComponentEnable(const std::string &name, LogLevel level)
Enable the logging output associated with that log component.
Definition: log.cc:302
Callback< R, Args... > MakeCallback(R(T::*memPtr)(Args...), OBJ objPtr)
Build Callbacks for class method members which take varying numbers of arguments and potentially retu...
Definition: callback.h:704
@ LOG_LEVEL_INFO
LOG_INFO and above.
Definition: log.h:104
cmd
Definition: second.py:40
#define list
bool verbose
void ReportRank(int color, MPI_Comm splitComm)
Report my rank, in both MPI_COMM_WORLD and the split communicator.
const int NS_COLOR
Tag for whether this rank should go into a new communicator ns-3 ranks will have color == 1.
const int NOT_NS_COLOR
Tag for whether this rank should go into a new communicator ns-3 ranks will have color == 1.
bool tracing
Flag to enable/disable generation of tracing files.
Ptr< PacketSink > sink
Pointer to the packet sink application.
Definition: wifi-tcp.cc:55