From cf6c4c83bbba36b67fe9693f77c4a4f76c11c5ff Mon Sep 17 00:00:00 2001 From: Paul Brown Date: Wed, 29 Jan 2014 19:07:37 -0600 Subject: [PATCH 1/4] Add new create_table_from_existing function --- README.md | 9 +++++++++ sqlaload/__init__.py | 2 +- sqlaload/schema.py | 14 ++++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1022772..e9017c3 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,14 @@ Here's the same example, but using the object-oriented API: table.upsert(item, ['key1', 'key2']) +Here's an example showing a table being copied from one database to another: + + from sqlaload import connect, create_table_from_existing + + srcEngine = connect('mysql+mysqldb://username:password@111.111.111.111/database') + destEngine = connect('mysql+mysqldb://username:password@localhost/database') + create_table_from_existing(srcEngine, 'oldTable', destEngine, 'newTable') + Functions --------- @@ -68,6 +76,7 @@ The library currently exposes the following functions: * ``connect(url)``, connect to a database and return an ``engine``. See the [SQLAlchemy documentation](http://docs.sqlalchemy.org/en/rel_0_8/core/engines.html#database-urls) for information about URL schemes and formats. * ``get_table(engine, table_name)`` will load a table configuration from the database, either reflecting the existing schema or creating a new table (with an ``id`` column). * ``create_table(engine, table_name)`` and ``load_table(engine, table_name)`` are more explicit than ``get_table`` but allow the same functions. +* ``create_table_from_existing(src_engine, src_table_name, dest_engine, dest_table_name)`` will clone the schema and create a table from a table in another database. * ``drop_table(engine, table_name)`` will remove an existing table, deleting all of its contents. * ``create_column(engine, table, column_name, type)`` adds a new column to a table, ``type`` must be a SQLAlchemy type class. * ``create_index(engine, table, columns)`` creates an index on the given table, based on a list of strings to specify the included ``columns``. diff --git a/sqlaload/__init__.py b/sqlaload/__init__.py index cdac3ed..381d188 100644 --- a/sqlaload/__init__.py +++ b/sqlaload/__init__.py @@ -1,5 +1,5 @@ from sqlaload.schema import connect -from sqlaload.schema import create_table, load_table, get_table, drop_table +from sqlaload.schema import create_table, create_table_from_existing, load_table, get_table, drop_table from sqlaload.schema import create_column from sqlaload.write import add_row, update_row from sqlaload.write import upsert, update, delete diff --git a/sqlaload/schema.py b/sqlaload/schema.py index 281acec..00e9837 100644 --- a/sqlaload/schema.py +++ b/sqlaload/schema.py @@ -38,6 +38,20 @@ def create_table(engine, table_name): engine._tables[table_name] = table return table +def create_table_from_existing(src_engine, src_table_name, dest_engine, dest_table_name): + with lock: + log.debug("Creating table: %s on %r" % (dest_table_name, dest_engine)) + src_engine._metadata.reflect(src_engine) + srcTable = Table(src_table_name, src_engine._metadata) + + destTable = Table(dest_table_name, dest_engine._metadata) + for column in srcTable.columns: + destTable.append_column(column.copy()) + + destTable.create() + dest_engine._tables[dest_table_name] = destTable + return destTable + def load_table(engine, table_name): with lock: log.debug("Loading table: %s on %r" % (table_name, engine)) From b8d5116baaf0dabec01ec25ccc0abd4ea9c59d4b Mon Sep 17 00:00:00 2001 From: Paul Brown Date: Wed, 29 Jan 2014 21:09:04 -0600 Subject: [PATCH 2/4] fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e9017c3..09303d4 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ include: * **Automatic schema**. If a column is written that does not exist on the table, it will be created automatically. -* **Upserts**. Records are either created or updated, depdending on +* **Upserts**. Records are either created or updated, depending on whether an existing version can be found. * **Query helpers** for simple queries such as all rows in a table or all distinct values across a set of columns. From d0219166499e0121f4b6cca9c020d785c05f8d3d Mon Sep 17 00:00:00 2001 From: Paul Brown Date: Wed, 29 Jan 2014 21:56:41 -0600 Subject: [PATCH 3/4] Fixes #33 --- setup.py | 3 +-- sqlaload/schema.py | 6 +----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 3d2a251..69da099 100644 --- a/setup.py +++ b/setup.py @@ -35,8 +35,7 @@ include_package_data=False, zip_safe=False, install_requires=[ - 'sqlalchemy>=0.7', - 'sqlalchemy-migrate>=0.7' + 'sqlalchemy>=0.7' ], tests_require=[], entry_points=\ diff --git a/sqlaload/schema.py b/sqlaload/schema.py index 00e9837..01eec5d 100644 --- a/sqlaload/schema.py +++ b/sqlaload/schema.py @@ -7,7 +7,6 @@ from sqlalchemy import Integer, UnicodeText, Float, DateTime, Boolean from sqlalchemy.schema import Table, MetaData, Column, Index from sqlalchemy.sql import and_, expression -from migrate.versioning.util import construct_engine log = logging.getLogger(__name__) lock = RLock() @@ -20,10 +19,7 @@ def connect(url): from sqlalchemy.pool import NullPool kw['poolclass'] = NullPool engine = create_engine(url, **kw) - engine = construct_engine(engine) - meta = MetaData() - meta.bind = engine - engine._metadata = meta + engine._metadata = MetaData(bind=engine) engine._tables = dict() engine._indexes = dict() return engine From 8f630593596eb67c4607dd321b3f7537445ac369 Mon Sep 17 00:00:00 2001 From: Paul Brown Date: Thu, 30 Jan 2014 02:20:20 -0600 Subject: [PATCH 4/4] added note about copying from mysql to postgresql --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 09303d4..4b83c57 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ The library currently exposes the following functions: * ``connect(url)``, connect to a database and return an ``engine``. See the [SQLAlchemy documentation](http://docs.sqlalchemy.org/en/rel_0_8/core/engines.html#database-urls) for information about URL schemes and formats. * ``get_table(engine, table_name)`` will load a table configuration from the database, either reflecting the existing schema or creating a new table (with an ``id`` column). * ``create_table(engine, table_name)`` and ``load_table(engine, table_name)`` are more explicit than ``get_table`` but allow the same functions. -* ``create_table_from_existing(src_engine, src_table_name, dest_engine, dest_table_name)`` will clone the schema and create a table from a table in another database. +* ``create_table_from_existing(src_engine, src_table_name, dest_engine, dest_table_name)`` will clone the schema and create a table from a table in another database. This is only likely to work if both the source database and destination database are the same type (mysql to postgresql won't work). * ``drop_table(engine, table_name)`` will remove an existing table, deleting all of its contents. * ``create_column(engine, table, column_name, type)`` adds a new column to a table, ``type`` must be a SQLAlchemy type class. * ``create_index(engine, table, columns)`` creates an index on the given table, based on a list of strings to specify the included ``columns``.