merging 0.4 branch to trunk. see CHANGES for details. 0.3 moves to maintenance branch in branches/rel_0_3.

11 files changed, 324 insertions, 110 deletions

diff --git a/test/perf/cascade_speed.py b/test/perf/cascade_speed.py
index d2e741442..34d046381 100644
--- a/test/perf/cascade_speed.py
+++ b/test/perf/cascade_speed.py
@@ -1,5 +1,7 @@
 import testbase
 from sqlalchemy import *
+from sqlalchemy.orm import *
+from testlib import *
 from timeit import Timer
 import sys
 
diff --git a/test/perf/masscreate.py b/test/perf/masscreate.py
index e603e2c00..346a725e3 100644
--- a/test/perf/masscreate.py
+++ b/test/perf/masscreate.py
@@ -1,8 +1,7 @@
 # times how long it takes to create 26000 objects
-import sys
-sys.path.insert(0, './lib/')
+import testbase
 
-from sqlalchemy.attributes import *
+from sqlalchemy.orm.attributes import *
 import time
 import gc
 
diff --git a/test/perf/masscreate2.py b/test/perf/masscreate2.py
index 3a68f3612..2e29a6327 100644
--- a/test/perf/masscreate2.py
+++ b/test/perf/masscreate2.py
@@ -1,11 +1,9 @@
-import sys
-sys.path.insert(0, './lib/')
-
+import testbase
 import gc
 
 import random, string
 
-from sqlalchemy.attributes import *
+from sqlalchemy.orm.attributes import *
 
 # with this test, run top.  make sure the Python process doenst grow in size arbitrarily.
 
diff --git a/test/perf/masseagerload.py b/test/perf/masseagerload.py
index 9d77fed54..f1c0f292b 100644
--- a/test/perf/masseagerload.py
+++ b/test/perf/masseagerload.py
@@ -1,62 +1,54 @@
-from testbase import PersistTest, AssertMixin
-import unittest, sys, os
-from sqlalchemy import *
-import StringIO
 import testbase
-import gc
-import time
-import hotshot
-import hotshot.stats
-
-db = testbase.db
+import hotshot, hotshot.stats
+from sqlalchemy import *
+from sqlalchemy.orm import *
+from testlib import *
 
 NUM = 500
 DIVISOR = 50
 
-class LoadTest(AssertMixin):
-    def setUpAll(self):
-        global items, meta,subitems
-        meta = MetaData(db)
-        items = Table('items', meta, 
-            Column('item_id', Integer, primary_key=True),
-            Column('value', String(100)))
-        subitems = Table('subitems', meta, 
-            Column('sub_id', Integer, primary_key=True),
-            Column('parent_id', Integer, ForeignKey('items.item_id')),
-            Column('value', String(100)))
-        meta.create_all()
-    def tearDownAll(self):
-        meta.drop_all()
-    def setUp(self):
-        clear_mappers()
+meta = MetaData(testbase.db)
+items = Table('items', meta, 
+              Column('item_id', Integer, primary_key=True),
+              Column('value', String(100)))
+subitems = Table('subitems', meta, 
+                 Column('sub_id', Integer, primary_key=True),
+                 Column('parent_id', Integer, ForeignKey('items.item_id')),
+                 Column('value', String(100)))
+
+class Item(object):pass
+class SubItem(object):pass
+mapper(Item, items, properties={'subs':relation(SubItem, lazy=False)})
+mapper(SubItem, subitems)
+
+def load():
+    global l
+    l = []
+    for x in range(1,NUM/DIVISOR + 1):
+        l.append({'item_id':x, 'value':'this is item #%d' % x})
+    #print l
+    items.insert().execute(*l)
+    for x in range(1, NUM/DIVISOR + 1):
         l = []
-        for x in range(1,NUM/DIVISOR + 1):
-            l.append({'item_id':x, 'value':'this is item #%d' % x})
+        for y in range(1, DIVISOR + 1):
+            z = ((x-1) * DIVISOR) + y
+            l.append({'sub_id':z,'value':'this is item #%d' % z, 'parent_id':x})
         #print l
-        items.insert().execute(*l)
-        for x in range(1, NUM/DIVISOR + 1):
-            l = []
-            for y in range(1, DIVISOR + 1):
-                z = ((x-1) * DIVISOR) + y
-                l.append({'sub_id':z,'value':'this is iteim #%d' % z, 'parent_id':x})
-            #print l
-            subitems.insert().execute(*l)    
-    def testload(self):
-        class Item(object):pass
-        class SubItem(object):pass
-        mapper(Item, items, properties={'subs':relation(SubItem, lazy=False)})
-        mapper(SubItem, subitems)
-        sess = create_session()
-        prof = hotshot.Profile("masseagerload.prof")
-        prof.start()
-        query = sess.query(Item)
-        l = query.select()
-        print "loaded ", len(l), " items each with ", len(l[0].subs), "subitems"
-        prof.stop()
-        prof.close()
-        stats = hotshot.stats.load("masseagerload.prof")
-        stats.sort_stats('time', 'calls')
-        stats.print_stats()
-        
-if __name__ == "__main__":
-    testbase.main()        
+        subitems.insert().execute(*l)    
+
+@profiling.profiled('masseagerload', always=True)
+def masseagerload(session):
+    query = session.query(Item)
+    l = query.select()
+    print "loaded ", len(l), " items each with ", len(l[0].subs), "subitems"
+
+def all():
+    meta.create_all()
+    try:
+        load()
+        masseagerload(create_session())
+    finally:
+        meta.drop_all()
+
+if __name__ == '__main__':
+    all()
diff --git a/test/perf/massload.py b/test/perf/massload.py
index 3530e4a65..92cf0fe92 100644
--- a/test/perf/massload.py
+++ b/test/perf/massload.py
@@ -1,13 +1,10 @@
-from testbase import PersistTest, AssertMixin
-import unittest, sys, os
-from sqlalchemy import *
-import sqlalchemy.orm.attributes as attributes
-import StringIO
 import testbase
-import gc
 import time
-
-db = testbase.db
+#import gc
+#import sqlalchemy.orm.attributes as attributes
+from sqlalchemy import *
+from sqlalchemy.orm import *
+from testlib import *
 
 NUM = 2500
 
@@ -20,7 +17,7 @@ for best results, dont run with sqlite :memory: database, and keep an eye on top
 class LoadTest(AssertMixin):
     def setUpAll(self):
         global items, meta
-        meta = MetaData(db)
+        meta = MetaData(testbase.db)
         items = Table('items', meta, 
             Column('item_id', Integer, primary_key=True),
             Column('value', String(100)))
@@ -28,8 +25,6 @@ class LoadTest(AssertMixin):
     def tearDownAll(self):
         items.drop()
     def setUp(self):
-        objectstore.clear()
-        clear_mappers()
         for x in range(1,NUM/500+1):
             l = []
             for y in range(x*500-500 + 1, x*500 + 1):
diff --git a/test/perf/massload2.py b/test/perf/massload2.py
index 1506ca503..d6424eb07 100644
--- a/test/perf/massload2.py
+++ b/test/perf/massload2.py
@@ -7,6 +7,7 @@ try:
 except:
     pass
 from sqlalchemy import *
+from testbase import Table, Column
 import time
 
 metadata = create_engine('sqlite://', echo=True)
diff --git a/test/perf/masssave.py b/test/perf/masssave.py
index 5690eac3f..dd03f3962 100644
--- a/test/perf/masssave.py
+++ b/test/perf/masssave.py
@@ -1,20 +1,16 @@
-from testbase import PersistTest, AssertMixin
-import unittest, sys, os
-from sqlalchemy import *
-import sqlalchemy.attributes as attributes
-import StringIO
 import testbase
-import gc
-import sqlalchemy.orm.session
 import types
-db = testbase.db
+from sqlalchemy import *
+from sqlalchemy.orm import *
+from testlib import *
+
 
 NUM = 250000
 
 class SaveTest(AssertMixin):
     def setUpAll(self):
         global items, metadata
-        metadata = MetaData(db)
+        metadata = MetaData(testbase.db)
         items = Table('items', metadata, 
             Column('item_id', Integer, primary_key=True),
             Column('value', String(100)))
diff --git a/test/perf/ormsession.py b/test/perf/ormsession.py
new file mode 100644
index 000000000..a9d310ef6
--- /dev/null
+++ b/test/perf/ormsession.py
@@ -0,0 +1,225 @@
+import testbase
+import time
+from datetime import datetime
+
+from sqlalchemy import *
+from sqlalchemy.orm import *
+from testlib import *
+from testlib.profiling import profiled
+
+class Item(object):
+    def __repr__(self):
+        return 'Item<#%s "%s">' % (self.id, self.name)
+class SubItem(object):
+    def __repr__(self):
+        return 'SubItem<#%s "%s">' % (self.id, self.name)
+class Customer(object):
+    def __repr__(self):
+        return 'Customer<#%s "%s">' % (self.id, self.name)
+class Purchase(object):
+    def __repr__(self):
+        return 'Purchase<#%s "%s">' % (self.id, self.purchase_date)
+
+items, subitems, customers, purchases, purchaseitems = \
+    None, None, None, None, None
+
+metadata = MetaData()
+
+@profiled('table')
+def define_tables():
+    global items, subitems, customers, purchases, purchaseitems
+    items = Table('items', metadata,
+                  Column('id', Integer, primary_key=True),
+                  Column('name', String(100)),
+                  test_needs_acid=True)
+    subitems = Table('subitems', metadata,
+                     Column('id', Integer, primary_key=True),
+                     Column('item_id', Integer, ForeignKey('items.id'),
+                            nullable=False),
+                     Column('name', String(100), PassiveDefault('no name')),
+                     test_needs_acid=True)
+    customers = Table('customers', metadata,
+                      Column('id', Integer, primary_key=True),
+                      Column('name', String(100)),
+                      *[Column("col_%s" % chr(i), String(64), default=str(i))
+                        for i in range(97,117)],
+                      **dict(test_needs_acid=True))
+    purchases = Table('purchases', metadata,
+                      Column('id', Integer, primary_key=True),
+                      Column('customer_id', Integer,
+                             ForeignKey('customers.id'), nullable=False),
+                      Column('purchase_date', DateTime,
+                             default=datetime.now),
+                      test_needs_acid=True)
+    purchaseitems = Table('purchaseitems', metadata,
+                      Column('purchase_id', Integer,
+                             ForeignKey('purchases.id'),
+                             nullable=False, primary_key=True),
+                      Column('item_id', Integer, ForeignKey('items.id'),
+                             nullable=False, primary_key=True),
+                      test_needs_acid=True)
+
+@profiled('mapper')
+def setup_mappers():
+    mapper(Item, items, properties={
+            'subitems': relation(SubItem, backref='item', lazy=True)
+            })
+    mapper(SubItem, subitems)
+    mapper(Customer, customers, properties={
+            'purchases': relation(Purchase, lazy=True, backref='customer')
+            })
+    mapper(Purchase, purchases, properties={
+            'items': relation(Item, lazy=True, secondary=purchaseitems)
+            })
+
+@profiled('inserts')
+def insert_data():
+    q_items = 1000
+    q_sub_per_item = 10
+    q_customers = 1000
+
+    con = testbase.db.connect()
+
+    transaction = con.begin()
+    data, subdata = [], []
+    for item_id in xrange(1, q_items + 1):
+        data.append({'name': "item number %s" % item_id})
+        for subitem_id in xrange(1, (item_id % q_sub_per_item) + 1):
+            subdata.append({'item_id': item_id,
+                         'name': "subitem number %s" % subitem_id})
+        if item_id % 100 == 0:
+            items.insert().execute(*data)
+            subitems.insert().execute(*subdata)
+            del data[:]
+            del subdata[:]
+    if data:
+        items.insert().execute(*data)
+    if subdata:
+        subitems.insert().execute(*subdata)
+    transaction.commit()
+
+    transaction = con.begin()
+    data = []
+    for customer_id in xrange(1, q_customers):
+        data.append({'name': "customer number %s" % customer_id})
+        if customer_id % 100 == 0:
+            customers.insert().execute(*data)
+            del data[:]
+    if data:
+        customers.insert().execute(*data)
+    transaction.commit()
+
+    transaction = con.begin()
+    data, subdata = [], []
+    order_t = int(time.time()) - (5000 * 5 * 60)
+    current = xrange(1, q_customers)
+    step, purchase_id = 1, 0
+    while current:
+        next = []
+        for customer_id in current:
+            order_t += 300
+            data.append({'customer_id': customer_id,
+                         'purchase_date': datetime.fromtimestamp(order_t)})
+            purchase_id += 1
+            for item_id in range(customer_id % 200, customer_id + 1, 200):
+                if item_id != 0:
+                    subdata.append({'purchase_id': purchase_id,
+                                    'item_id': item_id})
+            if customer_id % 10 > step:
+                next.append(customer_id)
+
+            if len(data) >= 100:
+                purchases.insert().execute(*data)
+                if subdata:
+                    purchaseitems.insert().execute(*subdata)
+                del data[:]
+                del subdata[:]
+        step, current = step + 1, next
+
+    if data:
+        purchases.insert().execute(*data)
+    if subdata:
+        purchaseitems.insert().execute(*subdata)
+    transaction.commit()
+
+@profiled('queries')
+def run_queries():
+    session = create_session()
+    # no explicit transaction here.
+    
+    # build a report of summarizing the last 50 purchases and 
+    # the top 20 items from all purchases
+
+    q = session.query(Purchase). \
+        limit(50).order_by(desc(Purchase.purchase_date)). \
+        options(eagerload('items'), eagerload('items.subitems'),
+                eagerload('customer'))
+
+    report = []
+    # "write" the report.  pretend it's going to a web template or something,
+    # the point is to actually pull data through attributes and collections.
+    for purchase in q:
+        report.append(purchase.customer.name)
+        report.append(purchase.customer.col_a)
+        report.append(purchase.purchase_date)
+        for item in purchase.items:
+            report.append(item.name)
+            report.extend([s.name for s in item.subitems])
+    
+    # mix a little low-level with orm
+    # pull a report of the top 20 items of all time
+    _item_id = purchaseitems.c.item_id
+    top_20_q = select([func.distinct(_item_id).label('id')],
+                      group_by=[purchaseitems.c.purchase_id, _item_id],
+                      order_by=[desc(func.count(_item_id)), _item_id],
+                      limit=20)
+    ids = [r.id for r in top_20_q.execute().fetchall()]
+    q2 = session.query(Item).filter(Item.id.in_(*ids))
+
+    for num, item in enumerate(q2):
+        report.append("number %s: %s" % (num + 1, item.name))
+
+@profiled('creating')
+def create_purchase():
+    # commit a purchase
+    customer_id = 100
+    item_ids = (10,22,34,46,58)
+
+    session = create_session()
+    session.begin()
+
+    customer = session.query(Customer).get(customer_id)
+    items = session.query(Item).filter(Item.id.in_(*item_ids))
+
+    purchase = Purchase()
+    purchase.customer = customer
+    purchase.items.extend(items)
+
+    session.flush()
+    session.commit()
+    session.expire(customer)
+
+def setup_db():
+    metadata.drop_all()
+    metadata.create_all()
+def cleanup_db():
+    metadata.drop_all()
+
+@profiled('default')
+def default():
+    run_queries()
+    create_purchase()
+
+@profiled('all')
+def main():
+    metadata.bind = testbase.db
+    try:
+        define_tables()
+        setup_mappers()
+        setup_db()
+        insert_data()
+        default()
+    finally:
+        cleanup_db()
+
+main()
diff --git a/test/perf/poolload.py b/test/perf/poolload.py
index d096f1c67..1a2ff6978 100644
--- a/test/perf/poolload.py
+++ b/test/perf/poolload.py
@@ -1,10 +1,11 @@
 # load test of connection pool
 
+import testbase
 from sqlalchemy import *
 import sqlalchemy.pool as pool
 import thread,time
-db = create_engine('mysql://scott:tiger@127.0.0.1/test', pool_timeout=30, echo_pool=True)
 
+db = create_engine(testbase.db.url, pool_timeout=30, echo_pool=True)
 metadata = MetaData(db)
 
 users_table = Table('users', metadata,
@@ -18,7 +19,7 @@ users_table.insert().execute([{'user_name':'user#%d' % i, 'password':'pw#%d' % i
 
 def runfast():
     while True:
-        c = db.connection_provider._pool.connect()
+        c = db.pool.connect()
         time.sleep(.5)
         c.close()
 #        result = users_table.select(limit=100).execute()
diff --git a/test/perf/threaded_compile.py b/test/perf/threaded_compile.py
index eb9e2f669..13ec31fd6 100644
--- a/test/perf/threaded_compile.py
+++ b/test/perf/threaded_compile.py
@@ -2,9 +2,12 @@
 when additional mappers are created while the existing 
 collection is being compiled."""
 
+import testbase
 from sqlalchemy import *
+from sqlalchemy.orm import *
 import thread, time
 from sqlalchemy.orm import mapperlib
+from testlib import *
 
 meta = MetaData('sqlite:///foo.db')
 
diff --git a/test/perf/wsgi.py b/test/perf/wsgi.py
index 365956dc7..d22eeb76a 100644
--- a/test/perf/wsgi.py
+++ b/test/perf/wsgi.py
@@ -1,53 +1,55 @@
 #!/usr/bin/python
+"""Uses ``wsgiref``, standard in Python 2.5 and also in the cheeseshop."""
 
+import testbase
 from sqlalchemy import *
-import sqlalchemy.pool as pool
+from sqlalchemy.orm import *
 import thread
-from sqlalchemy import exceptions
+from testlib import *
+
+port = 8000
 
 import logging
 logging.basicConfig()
 logging.getLogger('sqlalchemy.pool').setLevel(logging.INFO)
 
 threadids = set()
-#meta = MetaData('postgres://scott:tiger@127.0.0.1/test')
-
-#meta = MetaData('mysql://scott:tiger@localhost/test', poolclass=pool.SingletonThreadPool)
-meta = MetaData('mysql://scott:tiger@localhost/test')
+meta = MetaData(testbase.db)
 foo = Table('foo', meta, 
     Column('id', Integer, primary_key=True),
     Column('data', String(30)))
-
-meta.drop_all()
-meta.create_all()
-
-data = []
-for x in range(1,500):
-    data.append({'id':x,'data':"this is x value %d" % x})
-foo.insert().execute(data)
-
 class Foo(object):
     pass
-
 mapper(Foo, foo)
 
-root = './'
-port = 8000
+def prep():
+    meta.drop_all()
+    meta.create_all()
+
+    data = []
+    for x in range(1,500):
+        data.append({'id':x,'data':"this is x value %d" % x})
+    foo.insert().execute(data)
 
 def serve(environ, start_response):
+    start_response("200 OK", [('Content-type', 'text/plain')])
     sess = create_session()
     l = sess.query(Foo).select()
-            
-    start_response("200 OK", [('Content-type','text/plain')])
     threadids.add(thread.get_ident())
-    print "sending response on thread", thread.get_ident(), " total threads ", len(threadids)
-    return ["\n".join([x.data for x in l])]
+
+    print ("sending response on thread", thread.get_ident(),
+           " total threads ", len(threadids))
+    return [str("\n".join([x.data for x in l]))]
 
         
 if __name__ == '__main__':
-    from wsgiutils import wsgiServer
-    server = wsgiServer.WSGIServer (('localhost', port), {'/': serve})
-    print "Server listening on port %d" % port
-    server.serve_forever()
+    from wsgiref import simple_server
+    try:
+        prep()
+        server = simple_server.make_server('localhost', port, serve)
+        print "Server listening on port %d" % port
+        server.serve_forever()
+    finally:
+        meta.drop_all()