import boto
import boto.s3.connection
import sys
import pymongo
from pymongo import MongoClient
import os.path
import time



def file_exist(bucket, file_path):
	c =bucket.get_key(file_path)
	if c is not None and c.size > 0:
		return True
	return False

def exportItemForMongoCollection(obsId, db, bucket, log_file):	
	destination_collection =db[ 's3_'+obsId[:36]]
	source_collection = db[ obsId[:36]]
	print 'Start to exporting objectStore :%s '%obsId
	i = 0
	skip = 0
	last_percentage = 0
	total = source_collection.estimated_document_count()
	for item in source_collection.find(no_cursor_timeout=True):
		fs_path = item['fsPath']
		objectId = item['id']
		dest_item = destination_collection.find_one({'id':objectId})
		if dest_item is None:
			if os.path.isfile(fs_path):
				i += 1
				if not file_exist(bucket, '%s/%s'%(obsId,objectId)):
					key = bucket.new_key('%s/%s'%(obsId,objectId))
					try:
						key.set_contents_from_filename(fs_path)	
					except Exception as e:
						for k in range(10):
							try:
								time.sleep(10)
								key.set_contents_from_filename(fs_path)		
							except Exception as e:
								print "Error on saving object on S3"
								print e
								print "Sleeping 10 seconds and retry"
						
					item.pop('_id', None)
					item.pop('fsPath')
					item['uri'] = 's3://%s/%s'%(bucket.name, key.name)
					destination_collection.insert_one(item)
				if i % 100 == 0:
					print "Exported %i/%i   (skipped)%i "%(i, total, skip)
			else:
				log_file.writeline('Missing file for objectStoreid: %s ObjectId:%s path: %s'%(obsId, objectId, fs_path))
		else:
			skip += 1
			print "skipping Item"
		

def start_import(metadataCollection, bucket, log_file, done_file, skip_store):
	client = MongoClient()
	db = client['objectStore']
	metadataCollection = db[metadataCollection]
	print skip_store
	for item in metadataCollection.find(no_cursor_timeout=True):
		obsId = item['obsId']
		if obsId not in skip_store:
			destination_collection = db[ 's3_'+obsId[:36]]
			destination_collection.create_index([('id',pymongo.ASCENDING)])
			exportItemForMongoCollection(obsId, db, bucket, log_file)
			done_file.write('{}\n'.format(obsId))





if __name__=='__main__':
	args = sys.argv
	if not len(args) is not 3 :
		print "Error on applying script usage: python scriptName.py s3cfgPath objectstoreBucket"
	f = open(args[1])
	props = {}	
	for line in f:
		d =line.split('=')
		if len(d) == 2:
			props[d[0].strip()] = d[1].strip()
	skip_store =[]
	if os.path.isfile('store_done'):
		f = open('store_done')
		skip_store =[line.strip() for line in f if len(line) >0]

	bname = args[2]
	conn = boto.connect_s3(aws_access_key_id = props['access_key'], aws_secret_access_key = props['secret_key'],  host = props['host_base'], calling_format = boto.s3.connection.OrdinaryCallingFormat())
	bucket = conn.get_bucket(bname, validate=True)
	log_file = open('s3_migration.log', 'w')
	done_file = open('store_done_1', 'wb')
	start_import('metadataObjectStore',bucket, log_file, done_file, skip_store)
	log_file.close()