import pandas as pd import logging from thefirstock import thefirstock from utils.dbConnector import mySqlConnector from utils.config import DBConfig, GENRIC, ClientDetails import sys import time import os import pprint from datetime import datetime import pytz from decimal import Decimal for handler in logging.root.handlers[:]: logging.root.removeHandler(handler) class ISTFormatter(logging.Formatter): def formatTime(self, record, datefmt=None): # Set timezone to IST ist = pytz.timezone('Asia/Kolkata') record_time = datetime.fromtimestamp(record.created, tz=ist) return record_time.strftime(datefmt or '%Y-%m-%d %H:%M:%S') def convert_order_time(order_time_str): return datetime.strptime(order_time_str, '%H:%M:%S %d-%m-%Y').strftime('%Y-%m-%d %H:%M:%S') def elapsed_time(last_update_time): # Set timezone to IST ist = pytz.timezone('Asia/Kolkata') # If last_update_time is naive, localize it to IST if last_update_time.tzinfo is None: last_update_time = ist.localize(last_update_time) # Get the current time in IST current_time_ist = datetime.now(ist) # Calculate the elapsed time in seconds elapsed_time_seconds = (current_time_ist - last_update_time).total_seconds() return int(elapsed_time_seconds) class ClientManager: def __init__(self): self.logger = logging.getLogger(self.__class__.__name__) self.logger.setLevel(logging.INFO) self.logger.propagate = False formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') formatter = ISTFormatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') # Console handler console_handler = logging.StreamHandler() console_handler.setLevel(logging.INFO) console_handler.setFormatter(formatter) self.logger.addHandler(console_handler) # File handler file_handler = logging.FileHandler('/var/www/html/log/client_manager.log') file_handler.setLevel(logging.DEBUG) file_handler.setFormatter(formatter) self.logger.addHandler(file_handler) self.logger.info("Initializing ClientManager") self.db_connector = mySqlConnector() self.clientConfigDetails = ClientDetails.clientList def login(self, client): userId = client[0] password = client[1] TOTP = client[2] vendorCode = client[3] apiKey = client[4] self.logger.debug(f"userId: {userId}") self.logger.debug(f"TOTP: {TOTP}") self.logger.debug(f"vendorCode: {vendorCode}") self.logger.debug(f"apiKey: {apiKey}") self.logger.info("Attempting to login") login_response = thefirstock.firstock_login( userId=userId, password=password, TOTP=TOTP, vendorCode=vendorCode, apiKey=apiKey ) return login_response def storeClientPostition(self, clientID): self.logger.debug("Fetching position data") clientRootFolder = GENRIC.clientRootFolder.replace(GENRIC.replace_str, clientID ) positionInputFile = os.path.join(clientRootFolder, GENRIC.positionInputFile) positionProcessedFile = os.path.join(clientRootFolder, GENRIC.positionProcessedFile) try: position_book = thefirstock.firstock_PositionBook(clientID) if position_book['status'] == 'success': df = pd.DataFrame(position_book['data']) self.logger.debug("Position book data:\n%s", df) df.to_csv(positionInputFile, index=False) os.system(f'cp {positionInputFile} {positionProcessedFile}') else: self.logger.info(f"{clientID} - Position book data status is not success. {position_book['status']}") except Exception as e: self.logger.error("%s - An exception occurred while storing the position: %s", clientID, str(e)) def fetch_and_store_order_book(self, clientID, clientTableName): self.logger.debug(f"{clientID} - Fetching order book data") dataInsertedOrUpdated = False try: order_book = thefirstock.firstock_orderBook(clientID) if order_book['status'] == 'success': # Check if data count is more than zero if len(order_book['data']) > 0: # Convert data to DataFrame and log it df = pd.DataFrame(order_book['data']) df_sorted = df.sort_values(by='orderNumber') df_markdown = df_sorted.to_markdown(index=False) self.logger.debug("Order book data:\n%s", df_markdown) try: # Insert data into the database dataInsertedOrUpdated = self.db_connector.insertOrUpdateClientOrderBookData(clientTableName, df_sorted) except Exception as e: self.logger.error("An exception occurred while inserting order book data: %s", str(e)) else: self.logger.debug(f"Order book data status is success but empty. {order_book}") else: self.logger.error(f"Order book data status is NOT success. {order_book}") except Exception as e: self.logger.error("An exception occurred while fetching the order book: %s orderbook - %s", str(e), order_book) return dataInsertedOrUpdated def manageClients(self): self.clientDetails = [] self.logger.info("Initializing clientConfigDetails") for client in self.clientConfigDetails: clientID = client[0] self.logger.info(f"Checking for client {clientID}") clientTableName = GENRIC.clientTableName.replace(GENRIC.replace_str, clientID ) clientRootFolder = GENRIC.clientRootFolder.replace(GENRIC.replace_str, clientID ) os.system(f'mkdir -p {clientRootFolder}') os.system(f'chmod 777 {clientRootFolder}') os.chdir(clientRootFolder) login_response = self.login(client) try: if login_response['status'] == 'success': self.logger.info("Login successful") else: self.logger.error("Login failed: %s", login_response) continue except Exception as err: self.logger.error("Login failed: %s - exception - %s", login_response, err) continue tmp = {} tmp['clientID'] = clientID tmp['clientTableName'] = clientTableName tmp['clientDetails'] = client tmp['clientRootFolder'] = clientRootFolder self.clientDetails.append(tmp) self.logger.info("Adding Client : %s", clientID) self.db_connector.createClientOrderBookTable(clientTableName) while True: self.logger.debug("Get Master orderBook Data") self.db_connector = mySqlConnector(noLog=True) master_pd = self.db_connector.getCompleteOrderBookData(DBConfig.primaryOrderBookTable) #pprint.pprint(master_pd) if master_pd.empty: self.logger.debug("The master order book DataFrame is empty.") time.sleep(10) continue for client in self.clientDetails: clientID = client['clientID'] clientTableName = client['clientTableName'] clientDetails = client['clientDetails'] clientRootFolder = tmp['clientRootFolder'] os.chdir(clientRootFolder) self.logger.debug(f"{clientID} - Get Client orderBook Data") client_pd = self.db_connector.getCompleteOrderBookData(clientTableName) # Iterate through each row in master_pd master_pd = master_pd.sort_values(by='orderNumber') self.logger.debug("") self.logger.debug("-------------------------------------------") for index, master_row in master_pd.iterrows(): elapsed_time_secs = elapsed_time(master_row['lastUpdateTime']) self.logger.debug(f"{clientID}. Master OrderNumber: {master_row['orderNumber']} lastUpdateTime : {master_row['lastUpdateTime']} elapsed_time : {elapsed_time_secs}") # Do not process order where elapsed time is more than maxElapsedTimeAllowedToProcessOrders #if elapsed_time_secs > GENRIC.maxElapsedTimeAllowedToProcessOrders and int(master_row['id']) < 14 : if elapsed_time_secs > GENRIC.maxElapsedTimeAllowedToProcessOrders : self.logger.debug(f"{clientID} - Time elapsed({elapsed_time_secs} secs). skipping processing for OrderNumber: {master_row['orderNumber']}") continue else: self.logger.info(f"{clientID}. Master OrderNumber: {master_row['orderNumber']} lastUpdateTime : {master_row['lastUpdateTime']} elapsed_time : {elapsed_time_secs}") self.logger.info("Time check passed") # Define the condition to find the matching row in client_pd match_condition = (client_pd['OrderNumber'] == master_row['orderNumber']) & (client_pd['tradingSymbol'] == master_row['tradingSymbol']) & (client_pd['id'] == master_row['id']) # Check if there is a matching row in client_pd matching_row = client_pd[match_condition] if matching_row.empty: if master_row['status'] == 'OPEN' or master_row['status'] == 'COMPLETE': self.logger.info("") self.logger.info(f"{clientID} - +++ NEW ORDER - A") exchange = master_row['exchange'] tradingSymbol = master_row['tradingSymbol'] quantity = str(master_row['quantity']) priceType = master_row['priceType'] if priceType == 'MKT': price = '0.0' else: price = str(master_row['price']) product = master_row['product'] transactionType = master_row['transactionType'] priceType = master_row['priceType'] retention = master_row['retention'] triggerPrice = str(master_row['triggerPrice']) remarks = str(master_row['orderNumber']) userId = clientID ######################### # client Specific Hacks - handle these via config ######################### quantity = str(master_row['lotSize']) # buy only one lot if transactionType == 'B': price = str(master_row['price'] + Decimal('0.05')) # add 5 paise to original order else: price = str(master_row['price'] - Decimal('0.05')) # subtract 5 paise to original order self.logger.info(f"{clientID} - Placing order with parameters: exchange={exchange}, tradingSymbol={tradingSymbol}, quantity={quantity}, price={price}, product={product}, transactionType={transactionType}, priceType={priceType}, retention={retention}, triggerPrice={triggerPrice}, remarks={remarks}, userId={userId}") retVal = thefirstock.firstock_placeOrder( exchange=exchange, tradingSymbol=tradingSymbol, quantity=quantity, price=price, product=product, transactionType=transactionType, priceType=priceType, retention=retention, triggerPrice=triggerPrice, remarks=remarks, userId=userId ) try: if retVal['status'] == 'success': self.logger.info("Order successful. orderNumber - %s, requestTime - %s", retVal['data']['orderNumber'], retVal['data']['requestTime']) master_row['matchOrderNumber'] = retVal['data']['orderNumber'] master_row['matchOrderRequestTime'] = retVal['data']['requestTime'] master_row['matchOrderRequestTime'] = datetime.strptime(master_row['matchOrderRequestTime'], '%H:%M:%S %d-%m-%Y').strftime('%Y-%m-%d %H:%M:%S') else: self.logger.error("Order failed. reason: %s", retVal) continue except Exception as err: self.logger.error("Order failed. reason: %s. exception - %s", retVal, err) continue #insert the master row into client database with counter orderId appended self.logger.info(f"{clientID} - attempting to insert master specific order in client table") self.db_connector.insertClientOrderBookData(clientTableName, master_row) elif master_row['status'] == 'REJECTED' or master_row['status'] == 'CANCELED': self.logger.debug("") self.logger.debug(f"{clientID} - +++ NEW ORDER - B") self.logger.debug(f"{clientID} - master state {master_row['status']} for new row for OrderNumber: {master_row['orderNumber']}. Do nothing.") else: self.logger.info("") self.logger.info(f"{clientID} - +++ NEW ORDER - C") self.logger.error(f"{clientID} - TODO: unknown master state {master_row['status']} for new row for OrderNumber: {master_row['orderNumber']} in client_pd.") continue else: client_row = matching_row.iloc[0] differences = {} compare_columns = {'price', 'priceType', 'quantity', 'status'} for col in compare_columns: if col in master_pd.columns and col in client_pd.columns and master_row[col] != client_row[col]: differences[col] = {'master': master_row[col], 'client': client_row[col]} if differences: # Differences found, perform the update operation self.logger.info(f"{clientID} - Updating row for OrderNumber: {master_row['orderNumber']} in client_pd. Differences: {differences}") clientOrderNumber = str(int(client_row['matchOrderNumber'])) try: clientOrderNumberInInt = int(clientOrderNumber) client_order_matching_row = client_pd.loc[client_pd['OrderNumber'] == clientOrderNumberInInt] if client_order_matching_row.empty: self.logger.debug(f"No matching client order row found for OrderNumber: {clientOrderNumberInInt}") new_client_row = None # Or handle this case as needed else: new_client_row = client_order_matching_row.iloc[0] # Further processing with new_client_row if needed self.logger.info(f"Matching client order row found: {new_client_row.to_dict()}") except IndexError: self.logger.error(f"No row found for OrderNumber: {clientOrderNumberInInt} after filtering.") new_client_row = None except ValueError as ve: self.logger.error(f"ValueError encountered: {str(ve)}") new_client_row = None except Exception as e: self.logger.error(f"An unexpected error occurred: {str(e)}") new_client_row = None if new_client_row is not None and new_client_row['status'] in {'CANCELED', 'REJECTED', 'COMPLETE'}: self.logger.info(f"{clientID} - Differences found . But client corresponding order is not in processable state - new_client_row['status']") self.logger.info(f"{clientID} Update master row in client table to avoid further updates") self.db_connector.updateMasterRowInClientOrderBookData(clientTableName, master_row['orderNumber'], master_row['price'], master_row['priceType'], master_row['quantity'], master_row['status']) continue if master_row['status'] == 'CANCELED' and client_row['status'] == 'OPEN': self.logger.info("") self.logger.info(f"{clientID} - +++ MODIFY - CANCEL ORDER - CO") self.logger.info(f"{clientID} Attempt Cancel Order . orderNumber={master_row['orderNumber']} clientOrderNumber={clientOrderNumber}") retVal = thefirstock.firstock_cancelOrder( userId=clientID, orderNumber=clientOrderNumber ) try: if retVal['status'] == 'success': self.logger.info(f"{clientID} Cancel Order successful.") else: self.logger.error("Cancel Order failed. reason: %s", retVal) continue except Exception as e: self.logger.error("Cancel Order failed. reason: %s exception: %s", retVal, str(e)) continue self.logger.info(f"{clientID} Update master row in client table ") self.db_connector.updateMasterRowInClientOrderBookData(clientTableName, master_row['orderNumber'], master_row['price'], master_row['priceType'], master_row['quantity'], master_row['status']) elif master_row['status'] == 'CANCELED' and client_row['status'] == 'COMPLETE': self.logger.info("") self.logger.info(f"{clientID} - +++ MODIFY - ORPHAN ORDER") exchange = master_row['exchange'] tradingSymbol = master_row['tradingSymbol'] quantity = str(master_row['quantity']) price = str(master_row['price']) product = master_row['product'] transactionType = master_row['transactionType'] retention = master_row['retention'] triggerPrice = str(master_row['triggerPrice']) remarks = str('ORPHAN') userId = clientID ######################### # client Specific Hacks - handle these via config ######################### quantity = str(master_row['lotSize']) # buy only one lot price = str(master_row['price'] + Decimal('0.05')) priceType = 'MKT' if transactionType == 'B': transactionType = 'S' else: transactionType = 'B' self.logger.info(f"{clientID} - Placing ORPHAN order with parameters: exchange={exchange}, tradingSymbol={tradingSymbol}, quantity={quantity}, price={price}, product={product}, transactionType={transactionType}, priceType={priceType}, retention={retention}, triggerPrice={triggerPrice}, remarks={remarks}, userId={userId}") retVal = thefirstock.firstock_placeOrder( exchange=exchange, tradingSymbol=tradingSymbol, quantity=quantity, price=price, product=product, transactionType=transactionType, priceType=priceType, retention=retention, triggerPrice=triggerPrice, remarks=remarks, userId=userId ) try: if retVal['status'] == 'success': self.logger.info("Order successful. orderNumber - %s, requestTime - %s", retVal['data']['orderNumber'], retVal['data']['requestTime']) else: self.logger.error("Order failed. reason: %s", retVal) continue except Exception as err: self.logger.error("Order failed. reason: %s. exception - %s", retVal, err) continue self.logger.info(f"{clientID} Update master row in client table ") self.db_connector.updateMasterRowInClientOrderBookData(clientTableName, master_row['orderNumber'], master_row['price'], master_row['priceType'], master_row['quantity'], master_row['status']) elif master_row['priceType'] == 'MKT' and client_row['priceType'] != 'MKT': self.logger.info("") self.logger.info(f"{clientID} - +++ MODIFY - MKT ORDER") userId = clientID quantity = str(new_client_row['quantity']) price = str(new_client_row['price']) triggerPrice = str(new_client_row['triggerPrice']) exchange = new_client_row['exchange'] tradingSymbol = new_client_row['tradingSymbol'] priceType = 'MKT' self.logger.info(f"{clientID} - Modifying as Market order. parameters: exchange={exchange}, tradingSymbol={tradingSymbol}, quantity={quantity}, price={price}, priceType={priceType}, userId={userId}") retVal = thefirstock.firstock_ModifyOrder( userId=userId, orderNumber=clientOrderNumber, quantity=quantity, price=price, triggerPrice=triggerPrice, exchange=exchange, tradingSymbol=tradingSymbol, priceType=priceType ) try: if retVal['status'] == 'success': self.logger.info("Modify Order successful.") else: self.logger.error("Order failed. reason: %s", retVal) continue except Exception as err: self.logger.error("Order failed. reason: %s. exception - %s", retVal, err) continue self.logger.info(f"{clientID} Update master row in client table ") self.db_connector.updateMasterRowInClientOrderBookData(clientTableName, master_row['orderNumber'], master_row['price'], master_row['priceType'], master_row['quantity'], master_row['status']) elif master_row['status'] == 'COMPLETE' and client_row['status'] == 'OPEN': if new_client_row['status'] == 'OPEN': elapsed_time_secs = elapsed_time(master_row['lastUpdateTime']) self.logger.debug(f"{clientID}. Master OrderNumber: {master_row['orderNumber']} lastUpdateTime : {master_row['lastUpdateTime']} elapsed_time : {elapsed_time_secs}") # Do not perform market order where elapsed time is less than minWaitTimeToProcessClientMarketOrder if elapsed_time_secs < GENRIC.minWaitTimeToProcessClientMarketOrder : self.logger.debug(f"{clientID} - Time elapsed({elapsed_time_secs} secs). skipping processing for OrderNumber: {master_row['orderNumber']}") continue userId = clientID quantity = str(new_client_row['quantity']) price = str(new_client_row['price']) triggerPrice = str(new_client_row['triggerPrice']) exchange = new_client_row['exchange'] tradingSymbol = new_client_row['tradingSymbol'] priceType = 'MKT' self.logger.info(f"{clientID} - Modifying as Market order. parameters: exchange={exchange}, tradingSymbol={tradingSymbol}, quantity={quantity}, price={price}, priceType={priceType}, userId={userId}") retVal = thefirstock.firstock_ModifyOrder( userId=userId, orderNumber=clientOrderNumber, quantity=quantity, price=price, triggerPrice=triggerPrice, exchange=exchange, tradingSymbol=tradingSymbol, priceType=priceType ) try: if retVal['status'] == 'success': self.logger.info("Modify Order successful.") else: self.logger.error("Order failed. reason: %s", retVal) continue except Exception as err: self.logger.error("Order failed. reason: %s. exception - %s", retVal, err) continue self.logger.info(f"{clientID} Update master row in client table ") self.db_connector.updateMasterRowInClientOrderBookData(clientTableName, master_row['orderNumber'], master_row['price'], master_row['priceType'], master_row['quantity'], master_row['status']) else: self.logger.error(f"{clientID} Master order status is complete. but unhandled state for client - {new_client_row['status']}") elif master_row['price'] != client_row['price'] or master_row['quantity'] != client_row['quantity']: self.logger.info(f"{clientID} - todo : perform change of price and quantity in client") self.logger.info("") self.logger.info(f"{clientID} - +++ MODIFY - price and/or quantity") userId = clientID quantity = str(new_client_row['quantity']) price = str(new_client_row['price']) triggerPrice = str(new_client_row['triggerPrice']) exchange = new_client_row['exchange'] tradingSymbol = new_client_row['tradingSymbol'] priceType = new_client_row['priceType'] price = master_row['price'] quantity = master_row['quantity'] self.logger.info(f"{clientID} - Modifying as Market order. parameters: exchange={exchange}, tradingSymbol={tradingSymbol}, quantity={quantity}, price={price}, priceType={priceType}, userId={userId}") retVal = thefirstock.firstock_ModifyOrder( userId=userId, orderNumber=clientOrderNumber, quantity=quantity, price=price, triggerPrice=triggerPrice, exchange=exchange, tradingSymbol=tradingSymbol, priceType=priceType ) try: if retVal['status'] == 'success': self.logger.info("Modify Order successful.") else: self.logger.error("Order failed. reason: %s", retVal) continue except Exception as err: self.logger.error("Order failed. reason: %s. exception - %s", retVal, err) continue self.logger.info(f"{clientID} Update master row in client table ") self.db_connector.updateMasterRowInClientOrderBookData(clientTableName, master_row['orderNumber'], master_row['price'], master_row['priceType'], master_row['quantity'], master_row['status']) else: self.logger.error(f"{clientID} - todo : unknown state yet to be handled, if needed") # add code to update client orders. self.fetch_and_store_order_book(clientID, clientTableName) self.storeClientPostition(clientID) time.sleep(2) if __name__ == "__main__": client_manager = ClientManager() client_manager.manageClients()