Added support for new reports (#53)

Co-authored-by: Witchking <witchking@Witchkings-MacBook-Air.local>

README.md

@@ -114,18 +114,29 @@ print(de.get_historical_option_data(symbol="TATAMOTORS", start_date=start_date,
 
 ### Details
 
-| Name            | Module name                | Description                            | Argument                | Response |
-| --------------- | -------------------------- | -------------------------------------- | ----------------------- | -------- |
-| market activity | get_market_activity_report | get raw text of market activity report | date                    | string   |
-| bhav copy       | get_bhav_copy_zip          | download bhav copy zip for a given day | date, file_path_to_save | bool     |
+Note: Not all reports will be available for a given date, some reports are only available for a given date
+and some for 2-3 weeks.
+
+| Name                             | Module name                          | Description                                 | Argument            | Response          |
+| -------------------------------- | ------------------------------------ | ------------------------------------------- | ------------------- | ----------------- |
+| market activity                  | get_market_activity_report           | get raw text of market activity report      | date                | string            |
+| bhav copy                        | get_bhav_copy_zip                    | download bhav copy zip for a given day      | date, response_type | json or pandas df |
+| sec full bhav copy               | get_sec_full_bhav_copy               | download full bhav copy zip for a given day | date, response_type | json or pandas df |
+| volatitly report                 | get_volatility_report                | download volatility report for a given day  | date, response_type | json or pandas df |
+| fno participant wise oi data     | get_fno_participant_wise_oi_data     | fno participant oi data for a given day     | date, response_type | json or pandas df |
+| fno participant wise volume data | get_fno_participant_wise_volume_data | fno participant volume data for a given day | date, response_type | json or pandas df |
 
 ### step to run
 
 ```py
 from nsedt import reports as rep
 # date format "%d-%m-%Y"
 
-print(rep.get_market_activity_report(date="300424")) # format %d%m%y
-print(rep.get_bhav_copy_zip(date="30APR2024", file_path="path_where_you_want_to_save")) # format %d%b%Y
+print(rep.get_market_activity_report(date="30-05-2024"))
+print(rep.get_bhav_copy_zip(date="30-05-2024", response_type="json"))
+print(rep.get_sec_full_bhav_copy(date="30-05-2024", response_type="json"))
+print(rep.get_volatility_report(date="30-05-2024", response_type="json"))
+print(rep.get_fno_participant_wise_oi_data(date="30-05-2024", response_type="json"))
+print(rep.get_fno_participant_wise_volume_data(date="30-05-2024", response_type="json"))
 
 ```

nsedt/reports.py

@@ -6,23 +6,27 @@
 
 from nsedt.utils import get_cookies, fetch_csv, format_date, fetch_zip
 from nsedt.resources.constants import (
-    REPORT_URL, MARKET_ACTIVITY_REPORT, BHAV_COPY_REPORT)
+    REPORT_URL, MARKET_ACTIVITY_REPORT,
+    BHAV_COPY_REPORT, SEC_BHAV_COPY_REPORT, NSCCL_REPORTS, NSCCL_VOLT)
 
 log = logging.getLogger("root")
 
+# tbd
+# bulk deals
+# block deals
+
 
 def get_market_activity_report(date: str):
     """
     get_market_activity_report
 
     Args:\n
         date (str): date for which to download market activity report\n
-        response_type (str, Optional): define the response type panda_df | json . Default json\n
     Returns:
         string: string content of the file as right now its not possible 
         to format the content to json or pandas df
-    Expects: 
-        date to be in format of  "ddmmYY" eg: 30/04/2024 => 300424
+    Expects:
+        date to be in format of  "dd-mm-yyyy" eg: 30-04-2024
         all other cases will be invalidated
     """
     date = format_date(date, date_format='%d%m%y')
@@ -34,6 +38,29 @@ def get_market_activity_report(date: str):
     return fetch_csv(url, cookies, response_type="raw")
 
 
+def get_volatility_report(date: str, response_type: str="panda_df"):
+    """
+    get_volatility_report
+
+    Args:\n
+        date (str): date for which to download market activity report\n
+        response_type (str, Optional): define the response type panda_df | json . Default json\n
+    Returns:
+        string: string content of the file as right now its not possible
+        to format the content to json or pandas df
+    Expects:
+        date to be in format of  "dd-mm-yyyy" eg: 30-04-2024
+        all other cases will be invalidated
+    """
+    date = format_date(date, date_format='%d%m%Y')
+    if not date:
+        raise ValueError("Please provide date format in '%d-%m-%Y' format")
+    cookies = get_cookies()
+    url = f"{REPORT_URL}{NSCCL_VOLT}CMVOLT_{date}.CSV"
+    print(url)
+    return fetch_csv(url, cookies, response_type=response_type)
+
+
 def get_bhav_copy_zip(date: str, response_type: str="panda_df"):
     """
     get_market_activity_report
@@ -43,9 +70,8 @@ def get_bhav_copy_zip(date: str, response_type: str="panda_df"):
         path (str): path to save the bhav copy zip
     Returns:
         bool: if the file is save to the local path or not
-    Expects: 
-        date to be in format of  "ddmmYY" eg: 30/04/2024 => 30APR2024
-        all other cases will be invalidated
+    Expects:
+        date to be in format of  "dd-mm-yyyy" eg: 30-04-2024
     """
 
     date = format_date(date, date_format='%d%b%Y')
@@ -56,3 +82,72 @@ def get_bhav_copy_zip(date: str, response_type: str="panda_df"):
     url = f"{REPORT_URL}{BHAV_COPY_REPORT}{date[2:5]}/cm{date}bhav.csv.zip"
     file_name = url.split("/")[-1].replace(".zip", "")
     return fetch_zip(url, cookies, file_name=file_name, response_type=response_type)
+
+
+def get_sec_full_bhav_copy(date: str, response_type: str="panda_df"):
+    """
+    get_sec_full_bhav_copy
+
+    Args:\n
+        date (str): date for which to download market activity report\n
+        response_type (str, Optional): define the response type panda_df | json . Default json\n
+    Returns:
+        string: string content of the file as right now its not possible
+        to format the content to json or pandas df
+    Expects:
+        date to be in format of  "dd-mm-yyyy" eg: 30-04-2024
+        all other cases will be invalidated
+    """
+    date = format_date(date, date_format='%d%m%Y')
+    if not date:
+        raise ValueError("Please provide date format in '%d-%m-%Y' format")
+
+    cookies = get_cookies()
+    url = f"{REPORT_URL}{SEC_BHAV_COPY_REPORT}sec_bhavdata_full_{date}.csv"
+    return fetch_csv(url, cookies, response_type=response_type)
+
+
+def get_fno_participant_wise_oi_data(date: str, response_type: str="panda_df"):
+    """
+    fno_participant_wise_oi_data
+
+    Args:\n
+        date (str): date for which to download market activity report\n
+        response_type (str, Optional): define the response type panda_df | json . Default json\n
+    Returns:
+        string: string content of the file as right now its not possible
+        to format the content to json or pandas df
+    Expects:
+        date to be in format of  "ddmmYY" eg: 30/04/2024 => 300424
+        all other cases will be invalidated
+    """
+    date = format_date(date, date_format='%d%m%Y')
+    if not date:
+        raise ValueError("Please provide date format in '%d-%m-%Y' format")
+
+    cookies = get_cookies()
+    url = f"{REPORT_URL}{NSCCL_REPORTS}fao_participant_oi_{date}.csv"
+    return fetch_csv(url, cookies, response_type=response_type, skip_rows=1)
+
+
+def get_fno_participant_wise_volume_data(date: str,response_type: str="panda_df"):
+    """
+    get_fno_participant_wise_volume_data
+
+    Args:\n
+        date (str): date for which to download market activity report\n
+        response_type (str, Optional): define the response type panda_df | json . Default json\n
+    Returns:
+        string: string content of the file as right now its not possible
+        to format the content to json or pandas df
+    Expects:
+        date to be in format of  "ddmmYY" eg: 30/04/2024 => 300424
+        all other cases will be invalidated
+    """
+    date = format_date(date, date_format='%d%m%Y')
+    if not date:
+        raise ValueError("Please provide date format in '%d-%m-%Y' format")
+
+    cookies = get_cookies()
+    url = f"{REPORT_URL}{NSCCL_REPORTS}fao_participant_vol_{date}.csv"
+    return fetch_csv(url, cookies, response_type=response_type, skip_rows=1)

nsedt/resources/constants.py

@@ -43,3 +43,6 @@
 # Reports
 MARKET_ACTIVITY_REPORT = "archives/equities/mkt/MA"
 BHAV_COPY_REPORT = "content/historical/EQUITIES/2024/"
+SEC_BHAV_COPY_REPORT =  "products/content/"
+NSCCL_REPORTS = "content/nsccl/"
+NSCCL_VOLT = "archives/nsccl/volt/"

nsedt/utils/__init__.py

@@ -21,7 +21,15 @@
 
 
 
-
+def format_df(df):
+    """
+    Arg:\n
+        - df: pandas df
+    Reuturn:
+        - formatted df column 
+    """
+    df.columns = df.columns.str.lower().str.replace(' ','_').str.replace('\t','')
+    return df
 
 def format_date(input_string: str, date_format: str):
     """
@@ -37,7 +45,6 @@ def format_date(input_string: str, date_format: str):
         return None
 
 
-
 def get_headers():
     """
     Args:
@@ -49,8 +56,6 @@ def get_headers():
     return FakeHttpHeader().as_header_dict()
 
 
-
-
 def get_cookies():
     """
     Args:
@@ -154,8 +159,7 @@ def check_nd_convert(start_date: str, end_date: str) -> datetime:
     return start_date, end_date
 
 
-
-def fetch_csv(url, cookies, response_type="panda_df"):
+def fetch_csv(url, cookies, response_type="panda_df", skip_rows=None):
     """
     Args:
 
@@ -180,19 +184,19 @@ def fetch_csv(url, cookies, response_type="panda_df"):
         if response_type == "raw":
             return response.content
         csv_content = response.content.decode('utf-8')
-        df = pd.read_csv(io.StringIO(csv_content), error_bad_lines=False)
+        df = pd.read_csv(io.StringIO(csv_content), skiprows=skip_rows)
+        df = format_df(df)
         return df.to_json(orient='records') if response_type == "json" else df
     raise ValueError("Please try again in a minute.")
 
 
-def fetch_zip(url, cookies, file_name, response_type="panda_df"):
+def fetch_zip(url, cookies, file_name, response_type="panda_df", skip_rows=None):
     """
     Args:
 
         url (str): URL to fetch
         cookies (str): NSE cookies
         key (str, Optional):
-
     Returns:
 
         Pandas DataFrame: df generated from csv
@@ -217,6 +221,7 @@ def fetch_zip(url, cookies, file_name, response_type="panda_df"):
             except Exception as e:
                 raise ValueError("File not found in the zip folder.") from e
 
-            df = pd.read_csv(BytesIO(csv_content))
+            df = pd.read_csv(BytesIO(csv_content), skiprows=skip_rows)
+            df = format_df(df)
             return df.to_json(orient='records') if response_type == "json" else df
     raise ValueError("File might not be available this time or check your params")

tests/test_reports.py

@@ -3,9 +3,12 @@
 """
 
 import pandas as pd
-from nsedt.reports import get_bhav_copy_zip, get_market_activity_report
+from nsedt.reports import (
+    get_bhav_copy_zip, get_market_activity_report, get_sec_full_bhav_copy,
+    get_fno_participant_wise_volume_data, get_fno_participant_wise_oi_data,
+    get_volatility_report)
 
-# modify date to the latest date
+# modify date to the latest date while testing it out
 REPORT_DATE = "02-05-2024"
 
 
@@ -23,3 +26,35 @@ def test_get_bhav_copy_zip():
     """
     data = get_bhav_copy_zip(date=REPORT_DATE)
     assert isinstance(data, pd.DataFrame)
+
+
+def test_get_sec_full_bhav_copy():
+    """
+    Test get bhav copy report for a given date
+    """
+    data = get_sec_full_bhav_copy(date=REPORT_DATE)
+    assert isinstance(data, pd.DataFrame)
+
+
+def test_get_fno_participant_wise_volume_data():
+    """
+    Test get fno daily volume data participant wise
+    """
+    data = get_fno_participant_wise_volume_data(date=REPORT_DATE)
+    assert isinstance(data, pd.DataFrame)
+
+
+def test_get_fno_participant_wise_oi_data():
+    """
+    Test get fno daily oi data participant wise
+    """
+    data = get_fno_participant_wise_oi_data(date=REPORT_DATE)
+    assert isinstance(data, pd.DataFrame)
+
+
+def test_get_volatility_report():
+    """
+    Test get volatility report download
+    """
+    data = get_volatility_report(date=REPORT_DATE)
+    assert isinstance(data, pd.DataFrame)