Start line:  
End line:  

Snippet Preview

Snippet HTML Code

Stack Overflow Questions
  package com.orientechnologies.orient.graph.batch;
  
 
 import java.util.List;
 import java.util.Map;
this is and API for fast batch import of simple graphs, starting from an empty (or non existing) DB. This class allows import of graphs with
  • no properties on edges
  • no properties on vertices
  • Long values for vertex ids
These limitations are intended to have best performance on a very simple use case. If there limitations don't fit your requirements you can rely on other implementations (at the time of writing this they are planned, but not implemented yet) Typical usage: OGraphBatchInsertSimple batch = new OGraphBatchInsertSimple("plocal:your/db", "admin", "admin"); batch.begin(); batch.createEdge(0L, 1L); batch.createEdge(0L, 2L); ... batch.end(); There is no need to create vertices before connecting them: batch.createVertex(0L); batch.createVertex(1L); batch.createEdge(0L, 1L); is equivalent to (but less performing than) batch.createEdge(0L, 1L); batch.createVertex() is needed only if you want to create unconnected vertices.

Author(s):
Luigi Dell'Aquila (l.dellaquila-at-orientechnologies.com)
Since:
2.0 M3
 
 public class OGraphBatchInsertBasic {
 
   private final String        userName;
   private final String        dbUrl;
   private final String        password;
   Map<LongList<Long>>       out                      = new HashMap<LongList<Long>>();
   Map<LongList<Long>>       in                       = new HashMap<LongList<Long>>();
   private String              idPropertyName           = "uid";
   private String              edgeClass                = "E";
   private String              vertexClass              = "V";
   private ODatabaseDocumentTx db;
   private int                 averageEdgeNumberPerNode = -1;
   private int                 estimatedEntries         = -1;
   private int                 bonsaiThreshold          = 1000;
   private int[]               clusterIds;
   private long[]              lastClusterPositions;
   private long                last                     = 0;
   private boolean             walActive;
 
   private int                 parallel                 = 4;
   private AtomicInteger       runningThreads           = new AtomicInteger(0);
 
   class BatchImporterJob extends Thread {
 
     private final int mod;
     OClass            vClass;
 
     BatchImporterJob(int modOClass vertexClass) {
       this. = mod;
       this. = vertexClass;
     }
 
     @Override
     public void run() {
       try {
         ODatabaseDocumentTx db = new ODatabaseDocumentTx();
         db.open();
         db.declareIntent(new OIntentMassiveInsert());
         int clusterId = [];
 
         final String outField = "E".equals() ? "out_" : ("out_" + );
         final String inField = "E".equals() ? "in_" : ("in_" + );
 
         String clusterName = db.getStorage().getClusterById(clusterId).getName();
         // long firstAvailableClusterPosition = lastClusterPositions[mod] + 1;
 
        for (long i = i <= i += ) {
          final List<LongoutIds = .get(i);
          final List<LonginIds = .get(i);
          final ODocument doc = new ODocument();
          if (outIds == null && inIds == null) {
            db.save(docclusterName).delete();
          } else {
            doc.field(i);
            if (outIds != null) {
              final ORidBag outBag = new ORidBag();
              for (Long l : outIds) {
                final ORecordId rid = new ORecordId(getClusterId(l), getClusterPosition(l));
                outBag.add(rid);
              }
              doc.field(outFieldoutBag);
            }
            if (inIds != null) {
              final ORidBag inBag = new ORidBag();
              for (Long l : inIds) {
                final ORecordId rid = new ORecordId(getClusterId(l), getClusterPosition(l));
                inBag.add(rid);
              }
              doc.field(inFieldinBag);
            }
            db.save(docclusterName);
          }
        }
      } finally {
        synchronized () {
          .notifyAll();
        }
        .declareIntent(null);
        .close();
      }
    }
  }

  
Creates a new batch insert procedure by using admin user. It's intended to be used only for a single batch cycle (begin, create..., end)

Parameters:
iDbURL db connection URL (plocal:/your/db/path)
  public OGraphBatchInsertBasic(String iDbURL) {
    this. = iDbURL;
    this. = "admin";
    this. = "admin";
  }

  
Creates a new batch insert procedure. It's intended to be used only for a single batch cycle (begin, create..., end)

Parameters:
iDbURL db connection URL (plocal:/your/db/path)
iUserName db user name (use admin for new db)
iPassword db password (use admin for new db)
  public OGraphBatchInsertBasic(String iDbURLString iUserNameString iPassword) {
    this. = iDbURL;
    this. = iUserName;
    this. = iPassword;
  }

  
Creates the database (if it does not exist) and initializes batch operations. Call this once, before starting to create vertices and edges.
  public void begin() {
    if ()
    if ( > 0) {
    }
     = new ODatabaseDocumentTx();
    if (.exists()) {
      .open();
    } else {
      .create();
    }
     =  > 0 ? new HashMap<LongList<Long>>() : new HashMap<LongList<Long>>();
     =  > 0 ? new HashMap<LongList<Long>>() : new HashMap<LongList<Long>>();
    OClass vClass = .getMetadata().getSchema().getClass(this.);
    int[] existingClusters = vClass.getClusterIds();
    for (int c = existingClusters.lengthc <= c++) {
      vClass.addCluster(vClass.getName() + "_" + c);
    }
     = vClass.getClusterIds();
     = new long[.];
    for (int i = 0; i < .i++) {
      int clusterId = [i];
      try {
        [i] = .getStorage().getClusterById(clusterId).getLastPosition();
      } catch (Exception e) {
        throw new RuntimeException(e);
      }
    }
  }

  
Flushes data to db and closes the db. Call this once, after vertices and edges creation.
  public void end() {
    final OClass vClass = .getMetadata().getSchema().getClass();
    try {
      final String outField = "E".equals(this.) ? "out_" : ("out_" + this.);
      final String inField = "E".equals(this.) ? "in_" : ("in_" + this.);
      for (int i = 0; i <  - 1; i++) {
        Thread t = new BatchImporterJob(ivClass);
        t.start();
      }
      Thread t = new BatchImporterJob( - 1, vClass);
      t.run();
      if (.get() > 0) {
        synchronized () {
          while (.get() > 0) {
            try {
              .wait();
            } catch (InterruptedException e) {
            }
          }
        }
      }
    } finally {
      .declareIntent(null);
      .close();
      if ()
    }
  }

  
Creates a new vertex

Parameters:
v the vertex ID
  public void createVertex(final Long v) {
     =  < v ? v : ;
    final List<LongoutList = .get(v);
    if (outList == null) {
      .put(vnew ArrayList<Long>( <= 0 ? 4 : ));
    }
  }

  
Creates a new edge between two vertices. If vertices do not exist, they will be created

Parameters:
from id of the vertex that is starting point of the edge
to id of the vertex that is end point of the edge
  public void createEdge(final Long fromfinal Long to) {
    if (from < 0) {
      throw new IllegalArgumentException(" Invalid vertex id: " + from);
    }
    if (to < 0) {
      throw new IllegalArgumentException(" Invalid vertex id: " + to);
    }
     =  < from ? from : ;
     =  < to ? to : ;
    putInList(fromto);
    putInList(tofrom);
  }

  

Returns:
the configured average number of edges per node (optimization parameter, not calculated)
  public int getAverageEdgeNumberPerNode() {
  }

  
configures the average number of edges per node (for optimization). Use it before calling begin()

Parameters:
averageEdgeNumberPerNode
  public void setAverageEdgeNumberPerNode(final int averageEdgeNumberPerNode) {
    this. = averageEdgeNumberPerNode;
  }

  

Returns:
the property name where ids are written on vertices
  public String getIdPropertyName() {
    return ;
  }

  

Parameters:
idPropertyName the property name where ids are written on vertices
  public void setIdPropertyName(final String idPropertyName) {
    this. = idPropertyName;
  }

  

Returns:
the edge class name (E by default)
  public String getEdgeClass() {
    return ;
  }

  

Parameters:
edgeClass the edge class name
  public void setEdgeClass(final String edgeClass) {
    this. = edgeClass;
  }

  

Returns:
the vertex class name (V by default)
  public String getVertexClass() {
    return ;
  }

  

Parameters:
vertexClass the vertex class name
  public void setVertexClass(final String vertexClass) {
    this. = vertexClass;
  }

  

Returns:
the threshold for passing from emdedded RidBag to SBTreeBonsai (low level optimization).
  public int getBonsaiThreshold() {
    return ;
  }

  
Sets the threshold for passing from emdedded RidBag to SBTreeBonsai implementation, in number of edges (low level optimization). High values speed up writes but slow down reads later. Set -1 (default) to use default database configuration. See OGlobalConfiguration.RID_BAG_EMBEDDED_TO_SBTREEBONSAI_THRESHOLD}
  public void setBonsaiThreshold(final int bonsaiThreshold) {
    this. = bonsaiThreshold;
  }

  
Returns the estimated number of entries. 0 for auto-resize.
  public int getEstimatedEntries() {
    return ;
  }

  
Sets the estimated number of entries, 0 for auto-resize (default). This pre-allocate in memory structure avoiding resizing of them at run-time.
  public void setEstimatedEntries(final int estimatedEntries) {
    this. = estimatedEntries;
  }

  

Returns:
number of parallel threads used for batch import
  public int getParallel() {
    return ;
  }

  
sets the number of parallel threads to be used for batch insert

Parameters:
parallel number of threads (default 4)
  public void setParallel(int parallel) {
    this. = parallel;
  }
  private void putInList(final Long keyfinal Map<LongList<Long>> outfinal Long value) {
    List<Longlist = out.get(key);
    if (list == null) {
      list = new ArrayList<Long>( <= 0 ? 4 : );
      out.put(keylist);
    }
    list.add(value);
  }
  private void createBaseSchema() {
    final OSchema schema = .getMetadata().getSchema();
    OClass v;
    OClass e;
    if (!schema.existsClass("V")) {
      v = schema.createClass("V");
    } else {
      v = schema.getClass("V");
    }
    if (!schema.existsClass("E")) {
      e = schema.createClass("E");
    } else {
      e = schema.getClass("E");
    }
    if (!"V".equals(this.)) {
      if (!schema.existsClass(this.)) {
        schema.createClass(this.v);
      }
    }
    if (!schema.existsClass(this.)) {
      if (!schema.existsClass(this.)) {
        schema.createClass(this.e);
      }
    }
  }
  private long getClusterPosition(final long uid) {
    return [(int) (uid % )] + (uid / ) + 1;
  }
  private int getClusterId(final long left) {
    return [(int) (left % )];
  }
New to GrepCode? Check out our FAQ X