diff --git a/components/icons/CSV.js b/components/icons/CSV.js new file mode 100644 index 000000000..aa460c899 --- /dev/null +++ b/components/icons/CSV.js @@ -0,0 +1,17 @@ +import React from 'react'; + +const CSV = () => ( + + + + + + + + + + + +); + +export default CSV; \ No newline at end of file diff --git a/components/icons/Cypherl.js b/components/icons/Cypherl.js new file mode 100644 index 000000000..83d49320c --- /dev/null +++ b/components/icons/Cypherl.js @@ -0,0 +1,30 @@ +import React from 'react'; + +const Cypherl = () => ( + + + + + + + + + + + + + + + + + + + + + + + + +); + +export default Cypherl; \ No newline at end of file diff --git a/components/icons/Dremio.js b/components/icons/Dremio.js new file mode 100644 index 000000000..49bc9b27a --- /dev/null +++ b/components/icons/Dremio.js @@ -0,0 +1,29 @@ +import React from 'react'; + +const Dremio = () => ( + + + + + + + + + + + + + + + + + + + + + + + +); + +export default Dremio; \ No newline at end of file diff --git a/components/icons/DuckDB.js b/components/icons/DuckDB.js new file mode 100644 index 000000000..b95091764 --- /dev/null +++ b/components/icons/DuckDB.js @@ -0,0 +1,9 @@ +import React from 'react'; + +const DuckDB = () => ( + + + +); + +export default DuckDB; \ No newline at end of file diff --git a/components/icons/Kafka.js b/components/icons/Kafka.js new file mode 100644 index 000000000..fda564e0f --- /dev/null +++ b/components/icons/Kafka.js @@ -0,0 +1,9 @@ +import React from 'react'; + +const Kafka = () => ( + + + +); + +export default Kafka; \ No newline at end of file diff --git a/components/icons/MySQL.js b/components/icons/MySQL.js new file mode 100644 index 000000000..87728b56a --- /dev/null +++ b/components/icons/MySQL.js @@ -0,0 +1,9 @@ +import React from 'react'; + +const MySQL = () => ( + + + +); + +export default MySQL; \ No newline at end of file diff --git a/components/icons/Neo4j.js b/components/icons/Neo4j.js new file mode 100644 index 000000000..a3a1cbc21 --- /dev/null +++ b/components/icons/Neo4j.js @@ -0,0 +1,17 @@ +import React from 'react'; + +const Neo4j = () => ( + + + + + + + + + + + +); + +export default Neo4j; \ No newline at end of file diff --git a/components/icons/OracleDB.js b/components/icons/OracleDB.js new file mode 100644 index 000000000..5f1dedc35 --- /dev/null +++ b/components/icons/OracleDB.js @@ -0,0 +1,16 @@ +import React from 'react'; + +const OracleDB = () => ( + + + + + + + + + + +); + +export default OracleDB; \ No newline at end of file diff --git a/components/icons/Parquet.js b/components/icons/Parquet.js new file mode 100644 index 000000000..304e51f0e --- /dev/null +++ b/components/icons/Parquet.js @@ -0,0 +1,16 @@ +import React from 'react'; + +const Parquet = () => ( + + + + + + + + + + +); + +export default Parquet; \ No newline at end of file diff --git a/components/icons/PostgreSQL.js b/components/icons/PostgreSQL.js new file mode 100644 index 000000000..01d861cac --- /dev/null +++ b/components/icons/PostgreSQL.js @@ -0,0 +1,26 @@ +import React from 'react'; + +const PostgreSQL = () => ( + + + + + + + + + + + + + + + + + + + + +); + +export default PostgreSQL; \ No newline at end of file diff --git a/components/icons/Pulsar.js b/components/icons/Pulsar.js new file mode 100644 index 000000000..b6b45b245 --- /dev/null +++ b/components/icons/Pulsar.js @@ -0,0 +1,16 @@ +import React from 'react'; + +const Pulsar = () => ( + + + + + + + + + + +); + +export default Pulsar; \ No newline at end of file diff --git a/components/icons/SQLServer.js b/components/icons/SQLServer.js new file mode 100644 index 000000000..6775db433 --- /dev/null +++ b/components/icons/SQLServer.js @@ -0,0 +1,9 @@ +import React from 'react'; + +const SQLServer = () => ( + + + +); + +export default SQLServer; \ No newline at end of file diff --git a/components/icons/ServiceNow.js b/components/icons/ServiceNow.js new file mode 100644 index 000000000..92d535359 --- /dev/null +++ b/components/icons/ServiceNow.js @@ -0,0 +1,9 @@ +import React from 'react'; + +const ServiceNow = () => ( + + + +); + +export default ServiceNow; \ No newline at end of file diff --git a/components/icons/Spark.js b/components/icons/Spark.js new file mode 100644 index 000000000..b82aadf52 --- /dev/null +++ b/components/icons/Spark.js @@ -0,0 +1,18 @@ +import React from 'react'; + +const Spark = () => ( + + + + + + + + + + + + +); + +export default Spark; \ No newline at end of file diff --git a/components/icons/StarRocks.js b/components/icons/StarRocks.js new file mode 100644 index 000000000..b88860899 --- /dev/null +++ b/components/icons/StarRocks.js @@ -0,0 +1,19 @@ +import React from 'react'; + +const StarRocks = () => ( + + + + + + + + + + + + + +); + +export default StarRocks; \ No newline at end of file diff --git a/pages/data-migration/_meta.ts b/pages/data-migration/_meta.ts index 5fb61a803..973e858e5 100644 --- a/pages/data-migration/_meta.ts +++ b/pages/data-migration/_meta.ts @@ -1,11 +1,15 @@ export default { + "supported-data-sources": "Supported data sources", "best-practices": "Best practices", "csv": "CSV", "json": "JSON", "cypherl": "CYPHERL", "migrate-from-neo4j": "Migrate from Neo4j", + "migrate-from-neo4j-using-single-cypher-query": "Migrate from Neo4j using a single Cypher query", "migrate-from-rdbms": "Migrate from RDBMS using CSV files", "migrate-from-rdbms-directly": "Migrate from RDBMS using MAGE modules", + "migrate-iceberg-tables-from-data-lake-using-dremio": "Migrate Apache Iceberg tables from data lake using Dremio", + "migrate-with-apache-spark": "Migrate to Memgraph using Apache Spark", "migrate-memgraph-platform": "Migrate from Memgraph Platform to Memgraph MAGE", "export-data": "Export data" } diff --git a/pages/data-migration/migrate-from-neo4j-using-single-cypher-query.mdx b/pages/data-migration/migrate-from-neo4j-using-single-cypher-query.mdx new file mode 100644 index 000000000..6e6d4567e --- /dev/null +++ b/pages/data-migration/migrate-from-neo4j-using-single-cypher-query.mdx @@ -0,0 +1,90 @@ +--- +title: Migrate from Neo4j to Memgraph using a single Cypher query +description: Switch from Neo4j to Memgraph smoothly. Detailed documentation to guide the migration process for a seamless transition to Memgraph. +--- + +import { Cards } from 'nextra/components' +import { Callout } from 'nextra/components' +import {CommunityLinks} from '/components/social-card/CommunityLinks' + +# Migrating from Memgraph to Neo4j with a single cypher query + +## Overview +Migrating your graph database from **Neo4j** to **Memgraph** can be done seamlessly using a single Cypher query, +thanks to Memgraph’s built-in [**migrate** module](/advanced-algorithms/available-algorithms/migrate). +This approach eliminates the need for manual data exports and imports, making migration efficient and error-free. + +The **migrate** module is available directly in the `memgraph/memgraph-mage` Docker image, allowing you +to execute migration queries out of the box without additional installations. + +## Why use streaming instead of CSV Exports? +### Traditional export-import Challenges +Many database migrations rely on exporting data to CSV files and then importing them into the target system. +While this method is widely used, it has several drawbacks: +- **Time-consuming**: Large datasets require significant time for export and re-import. +- **Manual intervention**: CSV formatting issues often require additional preprocessing. + +### Benefits of streaming migration +With Memgraph’s `migrate` module, you can **stream data directly from Neo4j into Memgraph**, bypassing the need for intermediate files: +- **Instant data transfer**: No need to generate and handle large CSV files. +- **Automatic property mapping**: Ensures seamless migration of node and relationship properties. +- **Cypher Expressiveness**: By streaming rows into Memgraph's query engine, you can migrate data and recreate nodes and relationships +in the same fashion +- **Efficient for large graphs**: Can handle millions of nodes and relationships efficiently. +- **No downtime**: Data can be transferred while Memgraph remains operational. + +## Prerequisites +- **Neo4j** running on a specific Bolt port (e.g., `bolt://localhost:7687`) +- **Memgraph** running on a different Bolt port (e.g., `bolt://localhost:7688`) +- The **migrate** module available in the `memgraph/memgraph-mage` Docker image + +## Starting Neo4j and Memgraph +If you are running Neo4j and Memgraph on the same server, ensure they are running on different ports: + +```sh +# Start Neo4j (default port 7687) +# Start Memgraph on a different port (7688) with MAGE pre-installed +docker run -it --rm -p 7688:7687 memgraph/memgraph-mage +``` + +## Migrating the entire graph +To migrate the entire graph from Neo4j to Memgraph, use the following Cypher query in Memgraph: + +```cypher +CALL migrate.neo4j( + "MATCH (n)-[r]->(m) + RETURN properties(n) AS from_properties, + properties(r) AS edge_properties, + properties(m) AS to_properties", + {host: "localhost", port: 7687}) +YIELD row +RETURN row; +``` + +### Explanation +- The `MATCH (n)-[r]->(m)` query selects all nodes and relationships. +- The `RETURN` clause extracts node and relationship properties. +- The `{host: "localhost", port: 7687}` parameter specifies the Neo4j instance to migrate from. +- `YIELD row RETURN row` streams the data directly from Neo4j to Memgraph. + +## Migrating specific data +If you want to migrate only certain parts of the graph, use the following queries: + +### Migrate nodes with a specific label +```cypher +CALL migrate.neo4j(":Person", {host: "localhost", port: 7687}) YIELD row RETURN row; +``` +This migrates all nodes with the `Person` label. + +### Migrate relationships of a certain type +```cypher +CALL migrate.neo4j("[:KNOWS]", {host: "localhost", port: 7687}) YIELD row RETURN row; +``` +This migrates only relationships of type `KNOWS`. + +## Conclusion +Using Memgraph’s `migrate` module, you can efficiently migrate your graph data from Neo4j with a single Cypher +query. Whether you are migrating the full dataset or specific labels/relationships, this method allows for +**seamless, real-time streaming** without CSV exports. 🚀 + + diff --git a/pages/data-migration/migrate-from-neo4j.mdx b/pages/data-migration/migrate-from-neo4j.mdx index e76ca9189..292fadab5 100644 --- a/pages/data-migration/migrate-from-neo4j.mdx +++ b/pages/data-migration/migrate-from-neo4j.mdx @@ -5,6 +5,7 @@ description: Switch from Neo4j to Memgraph smoothly. Detailed documentation to g import { Cards } from 'nextra/components' import { Callout } from 'nextra/components' +import {CommunityLinks} from '/components/social-card/CommunityLinks' # Migrate from Neo4j to Memgraph @@ -552,3 +553,5 @@ nonsense or sales pitch, just tech. href="https://memgraph.com/office-hours" /> + + diff --git a/pages/data-migration/migrate-from-rdbms-directly.md b/pages/data-migration/migrate-from-rdbms-directly.md index 809e5574b..240fc42e0 100644 --- a/pages/data-migration/migrate-from-rdbms-directly.md +++ b/pages/data-migration/migrate-from-rdbms-directly.md @@ -6,10 +6,14 @@ description: Easily transition from RDBMS to Memgraph using MAGE modules. Our de # Migrate from RDBMS to Memgraph using MAGE modules This tutorial will help you import your data from a PostgreSQL database into Memgraph -directly using MAGE query modules. +directly using the [Memgraph MAGE `migrate` module](/advanced-algorithms/available-algorithms/migrate). +The migrate module contains a comprehensive list of data sources from which you're able to migrate your data +to Memgraph in one step. This migration tutorial makes migration from an external source to Memgraph possible in one less step than described in [migrating from a RDBMS to Memgraph using CSV files](/data-migration/migrate-from-rdbms). +The need for migrating directly from the source system arises from the fact that users so far needed to convert the +data from the source system to CSV files in order to migrate to Memgraph. Make sure you read both tutorials to see what fits your needs. In two of our blog posts, we've covered the [differences between relational and diff --git a/pages/data-migration/migrate-from-rdbms.mdx b/pages/data-migration/migrate-from-rdbms.mdx index b81dcda8a..92d57fc62 100644 --- a/pages/data-migration/migrate-from-rdbms.mdx +++ b/pages/data-migration/migrate-from-rdbms.mdx @@ -11,6 +11,15 @@ import { Steps } from 'nextra/components' This tutorial will help you import your data from a MySQL database into Memgraph using CSV files. + + +This way of migrating data into Memgraph requires exporting the data from the source relational +database system into CSV files, and then importing them into Memgraph. For direct data import +from a relational database system to Memgraph, please check out +our (guide for using migration modules inside Memgraph MAGE)[/data-migration/migrate-from-rdbms-directly]. + + + In two of our blog posts, we've covered the [differences between relational and graph database](https://memgraph.com/blog/graph-database-vs-relational-database) and outlined the [benefits of graph @@ -23,15 +32,6 @@ highly connected and require frequent retrieval with a flexible data model. If you're seeking a quick and reliable database that allows effortless modifications of data model and properties, a graph database is the way to go. - - -You can also use the `migrate` module which includes procedures such as -`migrate.mysql()`, `migrate.sql_server()`, and `migrate.oracle_db()` to transfer -and transform data into graph structures. For more details, see the [migrate -module documentation](/advanced-algorithms/available-algorithms/migrate). - - - ## Prerequisites To follow along, you will need: diff --git a/pages/data-migration/migrate-iceberg-tables-from-data-lake-using-dremio.mdx b/pages/data-migration/migrate-iceberg-tables-from-data-lake-using-dremio.mdx new file mode 100644 index 000000000..c5fa234f6 --- /dev/null +++ b/pages/data-migration/migrate-iceberg-tables-from-data-lake-using-dremio.mdx @@ -0,0 +1,168 @@ +--- +title: Migrate Iceberg tables from a Data Lake using Dremio +description: Memgraph is able to also query data lake files using Dremio as the query engine. This allows for graph analytics on top of data lake storage for seamless enablement of new graph use cases on your data. +--- + +import { Cards } from 'nextra/components' +import { Callout } from 'nextra/components' +import {CommunityLinks} from '/components/social-card/CommunityLinks' + +# Migrating data lake data from Apache Iceberg to Memgraph using Dremio + +## Overview +Migrating data from an **Apache Iceberg** table stored in a data lake to **Memgraph** can be efficiently done using **Dremio** as a query +engine and Memgraph’s **migrate module**. This setup eliminates the need for manual data exports and enables real-time data streaming into Memgraph. + +### Why Apache Iceberg? +Apache Iceberg is a high-performance table format designed for **data lakes**. It provides: +- **Schema evolution** without rewriting files. +- **Time travel** for querying historical data versions. +- **Partition evolution** to optimize queries. +- **ACID transactions** for reliability. + +### Why use Dremio? +Dremio acts as a **query engine** for your Iceberg data stored in **MinIO** or another object storage solution. It provides: +- **Federated querying** across multiple data sources. +- **SQL-based access** to Iceberg tables. +- **Optimized query performance** through Apache Arrow. + +### Benefits of using Arrow Flight for migration +Arrow Flight is a **high-performance data transport** built on **gRPC**. It enables: +- **Efficient data streaming** from Dremio to Memgraph. +- **Compression over the network**, reducing bandwidth usage. +- **Parallel execution** of queries for faster ingestion. +- **Schema-aware migration**, preserving types and structures. + +With Memgraph’s **migrate module**, we can directly stream **Apache Iceberg** data from Dremio into Memgraph **without using CSV exports**. + +--- + +## Setting up MinIO and Dremio +We will use **Docker Compose** to set up **MinIO** (for object storage) and **Dremio** (for querying the data lake). + +### Docker Compose configuration +```yaml +version: '3.8' + +services: + minio: + image: minio/minio + container_name: minio + environment: + MINIO_ROOT_USER: admin + MINIO_ROOT_PASSWORD: password + ports: + - "9000:9000" + - "9001:9001" + command: server /data --console-address ":9001" + volumes: + - minio_data:/data + + dremio: + image: dremio/dremio-oss + container_name: dremio + ports: + - "9047:9047" # Dremio Web UI + - "31010:31010" # JDBC port + - "32010:32010" # ODBC port + - "45678:45678" # Internal port + environment: + - DREMIO_JAVA_SERVER_EXTRA_OPTS=-Dservices.coordinator.enabled=true -Dservices.executor.enabled=true + - DREMIO_USERNAME=admin + - DREMIO_PASSWORD=admin + volumes: + - dremio_data:/opt/dremio/data + - dremio_conf:/opt/dremio/conf + +volumes: + minio_data: + dremio_data: + dremio_conf: +``` + +1. **Start MinIO and Dremio**: + ```sh + docker-compose up -d + ``` +2. **Access MinIO** at `http://localhost:9001` and create an **Iceberg** bucket. + +We will first create a bucket in the MinIO object storage. +![](/pages/data-migration/migrate-from-datalake-using-dremio/create-minio-bucket.png) + +We can see that the bucket is successfully added to MinIO +![](/pages/data-migration/migrate-from-datalake-using-dremio/observe-minio-bucket.png) + +3. **Connect MinIO to Dremio** through the **Dremio UI** at `http://localhost:9047`. + +Log in to Dremio: +![](/pages/data-migration/migrate-from-datalake-using-dremio/login-dremio.png) + +Create a MinIO datasource. MinIO is an S3-compatible object storage so the datasource selected is `Amazon S3`. +![](/pages/data-migration/migrate-from-datalake-using-dremio/create-minio-datasource.png) + +In order to connect to MinIO, we need to: +- tick the `Enable compatibility mode` +- add property `fs.s3a.path.style.access` and set it to `true` +- add property `fs.s3a.endpoint` and set it to `minio:9000` since that is our network name in docker compose +![](/pages/data-migration/migrate-from-datalake-using-dremio/additional-minio-datasource-options.png) + +For this example, we will create a dummy table using Dremio and populate it with 2 persons: +![](/pages/data-migration/migrate-from-datalake-using-dremio/querying-in-dremio.png) + + +--- + +## Migrating Iceberg data to Memgraph + +Once MinIO and Dremio are set up, and an **Iceberg table** (`persons`) is available, +we can use Memgraph’s **migrate module** to transfer the data. We will use the `Arrow Flight` +migration module since that enables fast transfer of Arrow compatible format using gRPC protocol. + +### Executing the Migration Query in Memgraph Lab +Run the following Cypher query in **Memgraph Lab**: + +```cypher +CALL migrate.arrow_flight( + "SELECT * FROM minio.iceberg.persons;", + { + host: "localhost", + port: 32010, + username: "admin", + password: "admin1!!" + } +) YIELD row +WITH row +CREATE (:Person {age: row.age, name: row.name}); +``` + +### Explanation: +- **`SELECT * FROM minio.iceberg.persons;`** → Queries the Iceberg table in **Dremio**. +- **Arrow Flight is used** for streaming the results efficiently into Memgraph. +- **Memgraph ingests** each row and creates **`Person`** nodes with `age` and `name` properties. + +Make sure that Memgraph sees the gRPC port exposed by Dremio for data transfer. + +--- + +## Visualizing the migration + +Once executed, you will see **nodes successfully ingested into Memgraph**: + +![](/pages/data-migration/migrate-from-datalake-using-dremio/migrate-from-dremio.png) + +--- + +## Conclusion +By using **Apache Iceberg**, **Dremio**, and **Memgraph’s migrate module**, we achieve: +- **Streaming data ingestion without CSV exports**. +- **Fast, compressed data transfers** over Arrow Flight. +- **Real-time migration from object storage** to Memgraph. +- **Seamless integration with modern data lakes**. + +## What's next? +Memgraph’s flexible migration capabilities allow you to **connect to any data source** and ingest data in real-time. 🚀 +For the list of supported data sources, please check our [migrate module](/advanced-algorithms/available-algorithms/migrate). + +Don't see an available data source? **Please contact us on Discord and we will enable you in the matter of days!** + + diff --git a/pages/data-migration/migrate-with-apache-spark.mdx b/pages/data-migration/migrate-with-apache-spark.mdx new file mode 100644 index 000000000..8a9ee2b3f --- /dev/null +++ b/pages/data-migration/migrate-with-apache-spark.mdx @@ -0,0 +1,177 @@ +--- +title: Migrate to Memgraph using Apache Spark +description: Migration of data using Apache Spark data processing engine. +--- + +import { Callout } from 'nextra/components' +import { Steps } from 'nextra/components' +import {CommunityLinks} from '/components/social-card/CommunityLinks' + +# High-speed graph ingestion with Memgraph and Apache Spark + +This guide demonstrates how to efficiently ingest 2 million edges into Memgraph within 3 seconds using Apache Spark. +The script leverages Memgraph's **IN_MEMORY_ANALYTICAL** mode for multithreading, **DROP GRAPH** for instant dataset +clearing, and **indexing** to optimize relationship insertions. Below, we break down the key components of the script and explain their importance. + +## Prerequisites +- Memgraph installed and running +- Apache Spark configured (using PySpark) +- this guide uses GQLAlchemy to query Memgraph with Python +- CSV files containing node and relationship data + +### Compatibility table + +When using Apache Spark with Memgraph, ensure that you use the correct Neo4j Spark Connector version based on +your Spark and Scala versions. Below is a compatibility table: + +| Spark Version | Artifact (Scala 2.12) | Artifact (Scala 2.13) | +|--------------|-------------------------------------------------|-------------------------------------------------| +| **3.3** | `org.neo4j:neo4j-connector-apache-spark_2.12:5.1.0_for_spark_3` | `org.neo4j:neo4j-connector-apache-spark_2.13:5.1.0_for_spark_3` | +| **3.2** | `org.neo4j:neo4j-connector-apache-spark_2.12:5.0.3_for_spark_3` | `org.neo4j:neo4j-connector-apache-spark_2.13:5.0.3_for_spark_3` | +| **3.0 and 3.1** | `org.neo4j:neo4j-connector-apache-spark_2.12:4.1.5_for_spark_3` | `org.neo4j:neo4j-connector-apache-spark_2.13:4.1.5_for_spark_3` | + +Make sure to include the appropriate dependency in your Spark configuration to ensure compatibility with Memgraph when +performing high-speed graph ingestion. 🚀- + +## Running Memgraph with Docker +To start Memgraph using Docker, use the following command: +```sh +docker run -it --rm -p 7687:7687 -p 3000:3000 memgraph/memgraph-mage +``` +This command runs Memgraph in an interactive mode, exposing its Bolt and web ports. + +## CSV file structure +Before running the ingestion script, ensure your CSV files are structured correctly: +- **node CSV (`pokec_medium_nodes.csv` -> 100k nodes)** + - Contains a single column: + - `id`: Unique identifier for each person + +- **relationship CSV (`pokec_medium_relationships.csv` -> 1.7 million edges)** + - Contains two columns: + - `from_id`: ID of the starting node (person) + - `to_id`: ID of the connected node (person) + +## Code breakdown + +### 1. Establish connection to Memgraph +```python +from gqlalchemy import Memgraph +from pyspark.sql import SparkSession +import time + +# Connection to Memgraph +url = "bolt://localhost:7687" +dbname = "memgraph" + +memgraph = Memgraph() +``` +Memgraph is initialized using the `gqlalchemy` Python client, establishing a connection via Bolt protocol. + +### 2. Configure Memgraph for optimal ingestion +```python +memgraph.execute("STORAGE MODE IN_MEMORY_ANALYTICAL;") +memgraph.execute("DROP GRAPH;") +memgraph.execute("CREATE INDEX ON :Person") +memgraph.execute("CREATE INDEX ON :Person(id)") +``` +#### Why `IN_MEMORY_ANALYTICAL` is important for multithreading +Setting **STORAGE MODE IN_MEMORY_ANALYTICAL** allows Memgraph to operate entirely in memory, maximizing data +processing speed and enabling efficient multi-threaded execution. This mode significantly improves performance for analytical queries and bulk inserts. + +#### How `DROP GRAPH` instantly clears the dataset +Executing **DROP GRAPH** removes all existing data in Memgraph instantaneously. This ensures that each ingestion +process starts with a clean slate, avoiding conflicts and ensuring consistent performance. + +#### Why indexing improves relationship ingestion +Creating indexes on the `Person` label and its `id` property speeds up node lookups when relationships are created. + Without indexing, the database would perform a full scan to find matching nodes, making relationship ingestion significantly slower. + +### 3. Initialize Spark session +```python +spark = ( + SparkSession.builder.config("neo4j.url", url) + .config("neo4j.authentication.basic.username", "") + .config("neo4j.authentication.basic.password", "") + .config( + "spark.jars.packages", + "org.neo4j:neo4j-connector-apache-spark_2.12:5.1.0_for_spark_3", + ) + .config("spark.driver.extraJavaOptions", "-Djava.security.manager=allow") + .config("spark.executor.extraJavaOptions", "-Djava.security.manager=allow") + .config("neo4j.database", dbname) + .getOrCreate() +) +``` +The Spark session is configured with the Memgraph connector to facilitate data ingestion. + +### 4. Load nodes from CSV and write to Memgraph +```python +csv_file_path = "pokec_medium_nodes.csv" +df = spark.read.csv(csv_file_path, header=True, inferSchema=True) + +node_ingest_start_time = time.time() +( + df.write.format("org.neo4j.spark.DataSource") + .mode("Overwrite") + .option("labels", "Person") + .option("node.keys", "id") + .option("batch.size", 5000) + .option("numPartitions", 10) + .save() +) +node_ingest_end_time = time.time() + +print(f"✅ Nodes successfully ingested into Memgraph from CSV after {(node_ingest_end_time - node_ingest_start_time):.2f}s!") +``` + +After running this part, we can observe the loading time: + +**✅ Nodes successfully ingested into Memgraph from CSV after 1.85s!** + +#### Optimizing node ingestion +- **Batch size of 5000**: Processes data in chunks to improve efficiency. +- **10 partitions**: Utilizes multiple threads for parallel processing, reducing ingestion time. +- **Overwrite mode**: Ensures a fresh start for the dataset. + +### 5. Load relationships and write to Memgraph +```python +relationships_csv_path = "pokec_medium_relationships.csv" +relationships_df = spark.read.csv(relationships_csv_path, header=True, inferSchema=True) + +query = """ +MATCH (p1:Person {id:event.from_id}) +MATCH (p2:Person {id:event.to_id}) +CREATE (p1)-[:CONNECTED_TO]->(p2) +""" + +rel_ingest_start_time = time.time() +( + relationships_df.write.format("org.neo4j.spark.DataSource") + .mode("Append") + .option("query", query) + .option("batch.size", 1000) + .option("numPartitions", 8) + .save() +) +rel_ingest_end_time = time.time() + +print(f"✅ Relationships successfully ingested into Memgraph after {(rel_ingest_end_time - rel_ingest_start_time):.2f}s!") +``` +#### Optimizing relationship ingestion +- **Batch size of 1000**: Ensures efficient transaction handling. +- **8 partitions**: Balances parallelism with system resources. +- **Using indexed node lookups**: The `MATCH` query relies on indexes for fast lookups. + +After ingesting the relationships, we can observe the loading part: + +**✅ Relationships successfully ingested into Memgraph after 3.10s!** + +## Conclusion +This script showcases how Memgraph, when combined with Spark, can rapidly ingest millions of edges in seconds by leveraging: +- **In-memory storage for parallel processing** +- **Efficient indexing for quick node lookups** +- **Multi-threaded Spark ingestion with batch processing** + +For further optimization, consider tuning batch sizes and partition numbers based on hardware resources. + + diff --git a/pages/data-migration/supported-data-sources.mdx b/pages/data-migration/supported-data-sources.mdx new file mode 100644 index 000000000..4d4357dce --- /dev/null +++ b/pages/data-migration/supported-data-sources.mdx @@ -0,0 +1,192 @@ +--- +title: Supported data sources +description: Discover the various data sources that can be integrated into Memgraph for a seamless graph computing experience. +--- + +import { Callout } from 'nextra/components' +import { Steps } from 'nextra/components' +import {CommunityLinks} from '/components/social-card/CommunityLinks' + +# Overview of supported data sources + +Memgraph supports multiple data sources for importing data into a running instance. Whether your data is structured in files, +relational databases, or other graph databases, Memgraph provides the flexibility to integrate and analyze your data efficiently. + +Memgraph supports file system imports like CSV files, offering efficient and structured data ingestion. **However, +if you want to migrate directly from another data source, you can use the +[`migrate` module](/advanced-algorithms/available-algorithms/migrate)** from Memgraph MAGE for a direct transition +without the need to convert your data into intermediate files. + + + +**In order to learn all the pre-requisites for importing data into Memgraph, check +[import best practices](/data-migration/best-practices).** + + + +## File types + +### CSV files + +CSV files provide a simple and efficient way to import tabular data into Memgraph +using the [LOAD CSV clause](/querying/clauses/load-csv). + +Using the [LOAD CSV import guide](/data-migration/csv), +you can implement your own CSV import through Memgraph LAB, or a simple database driver. + + +### JSON files + +Memgraph supports importing data from JSON files, allowing structured and semi-structured data to be +efficiently loaded, using the [`json_util` module](/advanced-algorithms/available-algorithms/json_util) +and [`import_util` module](/advanced-algorithms/available-algorithms/import_util). + +Check out the [JSON import guide](/data-migration/json). + +### Cypherl file +Cypherl file is a simple format which defines a cypher command per line. It is usually divided in 2 parts: +- `CREATE` statements for nodes, followed by +- `MATCH`, `MATCH` + `CREATE` statements for relationships + +An example for node creation in Cypherl would be similar to this query: +```cypher +CREATE (:Person {id: 1, name: "John", age: 18}); +CREATE (:Person {id: 2, name: "Mark", age: 25}); +``` + +While the example for relationship creation in Cypherl would be similar to this query: +```cypher +MATCH (p1:Person {id: 1}) MATCH (p2:Person {id: 2}) CREATE (p1)-[:KNOWS]->(p2); +``` + +If you have existing Cypherl files defining your graph structure, Memgraph can execute them directly to +populate your database. See the [Cypherl import guide](/data-migration/cypherl). + +## Database management systems + +### DuckDB + +Memgraph is fully compatible with all data source systems supported by DuckDB. +Using our [DuckDB migration module](/advanced-algorithms/available-algorithms/migrate#duckdb), +you can load data from any DuckDB-supported source, +run analytical queries directly on it, and import the results into Memgraph. + +- 📚 [List of DuckDB supported data sources](https://duckdb.org/docs/stable/data/data_sources.html) + +### MySQL + +Memgraph can integrate with MySQL databases, enabling seamless migration and real-time synchronization. +Learn more about [MySQL data import](/advanced-algorithms/available-algorithms/migrate#mysql). + +### PostgreSQL + +Memgraph can integrate with MySQL databases using the +[PostgreSQL migration module](/advanced-algorithms/available-algorithms/migrate#postgresql). + +### SQL Server + +For enterprises using Microsoft SQL Server, Memgraph supports data import using ETL processes or direct queries. +Read the [SQL Server import documentation](/advanced-algorithms/available-algorithms/migrate#sql_server). + +### StarRocks + +Memgraph supports migration from **StarRocks** through its compatibility with the +[MySQL migration module](/advanced-algorithms/available-algorithms/migrate#mysql). +Since StarRocks implements a MySQL-compatible protocol, you can use the same module to connect and migrate data into Memgraph. + +### OracleDB + +Memgraph can extract and import data from OracleDB, leveraging SQL queries or external connectors. +Find more details in the [OracleDB import guide](/advanced-algorithms/available-algorithms/migrate#oracle_db). + +## Graph databases + +### Neo4j + +Memgraph offers a dedicated [Neo4j migration module](/advanced-algorithms/available-algorithms/migrate#neo4j) +that allows users to easily transition from Neo4j. +By connecting to an existing Neo4j instance, you can migrate your entire graph to Memgraph using a single Cypher query. + +- 🛠️ [Step-by-step guide to migrate from Neo4j using a single Cypher query](/data-migration/migrate-from-neo4j-using-single-cypher-query) + +### Memgraph + +The fastest way to migrate from one Memgraph instance to another is by using **snapshot recovery**, +which leverages Memgraph’s proprietary durability files for efficient data backup and restore. +This process can be parallelized during startup for even faster recovery. + +- 💾 [Snapshot recovery documentation](/database-management/backup-and-restore#restore-data) + +If you only want to migrate a portion of your data, Memgraph also provides a **migration module** +that allows selective migration between Memgraph instances. + +- 🔄 [Memgraph migration module documentation](/advanced-algorithms/available-algorithms/migrate#memgraph) +- 🛠️ [Step-by-step guide to migrate from Neo4j using a single Cypher query (works for Memgraph with CALL migrate.memgraph())](/data-migration/migrate-from-neo4j-using-single-cypher-query) + +## RPC Protocols + +### Arrow Flight + +Memgraph supports migration from any system that uses **Arrow Flight** as its RPC protocol for fast and efficient +data transfer. The [Arrow Flight migration module](/advanced-algorithms/available-algorithms/migrate#arrow_flight) +enables you to stream data directly into Memgraph. + +## Streaming Sources + +### Apache Kafka + +Memgraph offers [native integration with **Apache Kafka**](/data-streams#kafka-and-redpanda), +enabling real-time data streaming directly into the graph. +It includes built-in Kafka consumers for easy setup and processing of incoming data streams. + +For more advanced setups, users can [integrate with **Kafka Connect**](/data-streams/kafka) or build custom consumers using +[Memgraph client libraries](/client-libraries) available in multiple programming languages. + +### Apache Pulsar + +Memgraph offers [native integration with **Apache Pulsar**](/data-streams#pulsar), +enabling real-time data streaming directly into the graph. +It includes built-in Pulsar consumers for easy setup and processing of incoming data streams. + +For more advanced setups, users can build custom consumers using +[Memgraph client libraries](/client-libraries) available in multiple programming languages. + +### Apache Spark + +Memgraph is compatible with the **Neo4j Spark Connector**, which makes it easy to integrate Spark-powered +pipelines with Memgraph. You can use this connector to process large-scale data in Spark and ingest the results +directly into Memgraph for real-time graph analytics. + +- ⚡ [Step-by-step guide to migrate with Apache Spark](/data-migration/migrate-with-apache-spark) + +### Dremio + +Memgraph supports migration from the **Dremio** query engine using the +[Arrow Flight RPC protocol](/advanced-algorithms/available-algorithms/migrate#arrow_flight), which enables +high-performance data transfer from data lakes and lakehouses. You can use Dremio to query formats like +Apache Iceberg and stream the results directly into Memgraph. + +- 🧊 [Guide to migrate Iceberg tables from a data lake using Dremio](/data-migration/migrate-iceberg-tables-from-data-lake-using-dremio) + +## Data platforms + +### ServiceNow + +Memgraph supports the [migration module from ServiceNow](/advanced-algorithms/available-algorithms/migrate#servicenow) +through its dedicated migration module. By connecting to the **ServiceNow REST API**, the module enables you to +retrieve and ingest data directly into Memgraph for further analysis and visualization. + +## Storage services + +### Amazon S3 + +Memgraph supports data migration from **Amazon S3** and other **S3-compatible storage services**. +Using the [S3 migration module](/advanced-algorithms/available-algorithms/migrate#s3), +you can load structured data stored in cloud object storage directly into Memgraph for graph-based processing and analysis. + + + +If you're unsure about the best way to import your data or need assistance, reach out on [Discord](https://discord.com/invite/memgraph) for help. + + + \ No newline at end of file diff --git a/pages/index.mdx b/pages/index.mdx index f5b27eba0..24dad5a4e 100644 --- a/pages/index.mdx +++ b/pages/index.mdx @@ -4,18 +4,33 @@ description: Discover the comprehensive Memgraph documentation and learn how to --- import { Cards } from 'nextra/components' +import CSV from '/components/icons/CSV' import CSharp from '/components/icons/CSharp' import Cpp from '/components/icons/Cpp' +import Cypherl from '/components/icons/Cypherl' +import Dremio from '/components/icons/Dremio' +import DuckDB from '/components/icons/DuckDB' import Go from '/components/icons/Go' import Graphql from '/components/icons/Graphql' import Haskell from '/components/icons/Haskell' import Java from '/components/icons/Java' import JavaScript from '/components/icons/JavaScript' +import Kafka from '/components/icons/Kafka' +import MySQL from '/components/icons/MySQL' +import Neo4j from '/components/icons/Neo4j' import NodeJS from '/components/icons/NodeJS' +import OracleDB from '/components/icons/OracleDB' +import Parquet from '/components/icons/Parquet' import PHP from '/components/icons/PHP' +import PostgreSQL from '/components/icons/PostgreSQL' +import Pulsar from '/components/icons/Pulsar' import Python from '/components/icons/Python' import Ruby from '/components/icons/Ruby' import Rust from '/components/icons/Rust' +import ServiceNow from '/components/icons/ServiceNow' +import Spark from '/components/icons/Spark' +import SQLServer from '/components/icons/SQLServer' +import StarRocks from '/components/icons/StarRocks' import {CommunityLinks} from '/components/social-card/CommunityLinks' # Memgraph Documentation @@ -135,6 +150,8 @@ libraries and follow their getting started guide. You can migrate your data from an existing graph or SQL database using CSV or JSON files, and import data using queries within a CYPHERL file. +### Migration guides + + + +### Supported source systems + + + } + title="CSV" + href="/data-migration/supported-data-sources#csv-files" + /> + } + title="JSON" + href="/data-migration/supported-data-sources#json-files" + /> + } + title="Cypherl" + href="/data-migration/supported-data-sources#cypherl-file" + /> + } + title="Parquet" + href="/data-migration/supported-data-sources#duckdb" + /> + } + title="Neo4j" + href="/data-migration/supported-data-sources#neo4j" + /> + } + title="DuckDB" + href="/data-migration/supported-data-sources#duckdb" + /> + } + title="MySQL" + href="/data-migration/supported-data-sources#mysql" + /> + } + title="PostgreSQL" + href="/data-migration/supported-data-sources#postgresql" + /> + } + title="OracleDB" + href="/data-migration/supported-data-sources#oracledb" + /> + } + title="StarRocks" + href="/data-migration/supported-data-sources#starrocks" + /> + } + title="SQL Server" + href="/data-migration/supported-data-sources#sql-server" + /> + } + title="Apache Spark" + href="/data-migration/supported-data-sources#apache-spark" + /> + } + title="Dremio" + href="/data-migration/supported-data-sources#dremio" + /> + } + title="ServiceNow" + href="/data-migration/supported-data-sources#servicenow" + /> + } + title="Apache Kafka" + href="/data-migration/supported-data-sources#apache-kafka" + /> + } + title="Apache Pulsar" + href="/data-migration/supported-data-sources#apache-pulsar" + /> + + ## Query and analyze data You can query Memgraph using Cypher query language, use algorithms available in diff --git a/pages/memgraph-in-production/general-suggestions.mdx b/pages/memgraph-in-production/general-suggestions.mdx index df4afdd15..ab6c73e25 100644 --- a/pages/memgraph-in-production/general-suggestions.mdx +++ b/pages/memgraph-in-production/general-suggestions.mdx @@ -337,6 +337,9 @@ you're working with CSV files, JSON streams, Kafka topics, or external data sour We strongly encourage users to review the [best practices for data migration](/data-migration/best-practices), which cover recommendations and tips to ensure a reliable and performant data import process tailored to Memgraph. +If you have your data in a different source system, check out our guide on +[how to migrate your data to Memgraph from different source systems](/data-migration/supported-data-sources). + ## Enterprise features you might require Memgraph provides a rich set of **enterprise-grade features** designed to support production workloads at scale. These include: diff --git a/public/pages/data-migration/migrate-from-datalake-using-dremio/additional-minio-datasource-options.png b/public/pages/data-migration/migrate-from-datalake-using-dremio/additional-minio-datasource-options.png new file mode 100644 index 000000000..1d21a7f6c Binary files /dev/null and b/public/pages/data-migration/migrate-from-datalake-using-dremio/additional-minio-datasource-options.png differ diff --git a/public/pages/data-migration/migrate-from-datalake-using-dremio/create-minio-bucket.png b/public/pages/data-migration/migrate-from-datalake-using-dremio/create-minio-bucket.png new file mode 100644 index 000000000..be185d3c7 Binary files /dev/null and b/public/pages/data-migration/migrate-from-datalake-using-dremio/create-minio-bucket.png differ diff --git a/public/pages/data-migration/migrate-from-datalake-using-dremio/create-minio-datasource.png b/public/pages/data-migration/migrate-from-datalake-using-dremio/create-minio-datasource.png new file mode 100644 index 000000000..37d8ebff7 Binary files /dev/null and b/public/pages/data-migration/migrate-from-datalake-using-dremio/create-minio-datasource.png differ diff --git a/public/pages/data-migration/migrate-from-datalake-using-dremio/login-dremio.png b/public/pages/data-migration/migrate-from-datalake-using-dremio/login-dremio.png new file mode 100644 index 000000000..8e5c124dc Binary files /dev/null and b/public/pages/data-migration/migrate-from-datalake-using-dremio/login-dremio.png differ diff --git a/public/pages/data-migration/migrate-from-datalake-using-dremio/migrate-from-dremio.png b/public/pages/data-migration/migrate-from-datalake-using-dremio/migrate-from-dremio.png new file mode 100644 index 000000000..91d972df2 Binary files /dev/null and b/public/pages/data-migration/migrate-from-datalake-using-dremio/migrate-from-dremio.png differ diff --git a/public/pages/data-migration/migrate-from-datalake-using-dremio/observe-minio-bucket.png b/public/pages/data-migration/migrate-from-datalake-using-dremio/observe-minio-bucket.png new file mode 100644 index 000000000..e47db25cd Binary files /dev/null and b/public/pages/data-migration/migrate-from-datalake-using-dremio/observe-minio-bucket.png differ diff --git a/public/pages/data-migration/migrate-from-datalake-using-dremio/querying-in-dremio.png b/public/pages/data-migration/migrate-from-datalake-using-dremio/querying-in-dremio.png new file mode 100644 index 000000000..925887f5e Binary files /dev/null and b/public/pages/data-migration/migrate-from-datalake-using-dremio/querying-in-dremio.png differ