chore: initialize the project

.editorconfig

@@ -0,0 +1,14 @@
+# top-most EditorConfig file
+root = true
+
+# Unix-style newlines with a newline ending every file
+[*]
+end_of_line = lf
+insert_final_newline = true
+
+# Matches multiple files with brace expansion notation
+# Set default charset
+[*]
+charset = utf-8
+indent_style = space
+indent_size = 2

.github/workflows/githubhfsync.yaml

@@ -0,0 +1,26 @@
+name: Sync Repository to HuggingFace Space
+
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:  # Enable manual trigger
+
+jobs:
+  sync-to-huggingface:
+    name: Sync code to HuggingFace Space
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0  # Fetch all history for all branches and tags
+          lfs: true      # Enable Git LFS support
+
+      - name: Push to HuggingFace Space
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: |
+          if ! git push https://lamhieu:[email protected]/spaces/lamhieu/docsifer main; then
+            echo "Failed to sync with HuggingFace Space"
+            exit 1
+          fi

.gitignore

@@ -0,0 +1,54 @@
+# Python bytecode files
+# Ignore Python bytecode files
+*.pyc
+
+# Distribution packages
+# Ignore distribution packages
+/dist/*
+
+# Test and coverage reports
+# Ignore coverage and test result files
+.coverage
+.pytest_cache
+.mypy_cache
+
+# Log and temporary files
+# Ignore log files and temporary files
+*.log
+*.tmp
+tmp
+
+# System files
+# Ignore OS generated files
+.DS_Store
+
+# IDE and editor specific files
+# Ignore project-specific files from various IDEs and editors
+.idea/*
+.vscode/*
+.python-version
+
+# Generated documentation
+# Ignore generated documentation files
+/docs/site/*
+
+# Virtual environments
+# Ignore virtual environment directories
+.venv
+
+# Configuration files
+# Ignore configuration files
+.poetry.toml
+.env.local
+.env.development
+.env.test
+.env.production
+.env
+
+# Temporary files and directories for operations
+# Ignore Ops temporary files and directories
+.aider*
+
+# Credentials and secrets
+# Ignore credentials and secrets files
+.credentials

.pylintrc

@@ -0,0 +1,579 @@
+[MAIN]
+
+# Specify a configuration file.
+#rcfile=
+
+# Python code to execute, usually for sys.path manipulation such as
+# pygtk.require().
+#init-hook=
+
+# Files or directories to be skipped. They should be base names, not
+# paths.
+ignore=CVS
+
+# Add files or directories matching the regex patterns to the ignore-list. The
+# regex matches against paths and can be in Posix or Windows format.
+ignore-paths=
+
+# Files or directories matching the regex patterns are skipped. The regex
+# matches against base names, not paths.
+ignore-patterns=^\.#
+
+# Pickle collected data for later comparisons.
+persistent=yes
+
+# List of plugins (as comma separated values of python modules names) to load,
+# usually to register additional checkers.
+load-plugins=
+    pylint.extensions.check_elif,
+    pylint.extensions.bad_builtin,
+    pylint.extensions.docparams,
+    pylint.extensions.for_any_all,
+    pylint.extensions.set_membership,
+    pylint.extensions.code_style,
+    pylint.extensions.overlapping_exceptions,
+    pylint.extensions.typing,
+    pylint.extensions.redefined_variable_type,
+    pylint.extensions.comparison_placement,
+    pylint.extensions.mccabe,
+
+# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
+# number of processors available to use.
+jobs=0
+
+# When enabled, pylint would attempt to guess common misconfiguration and emit
+# user-friendly hints instead of false-positive error messages.
+suggestion-mode=yes
+
+# Allow loading of arbitrary C extensions. Extensions are imported into the
+# active Python interpreter and may run arbitrary code.
+unsafe-load-any-extension=no
+
+# A comma-separated list of package or module names from where C extensions may
+# be loaded. Extensions are loading into the active Python interpreter and may
+# run arbitrary code
+extension-pkg-allow-list=
+
+# Minimum supported python version
+py-version = 3.7.2
+
+# Control the amount of potential inferred values when inferring a single
+# object. This can help the performance when dealing with large functions or
+# complex, nested conditions.
+limit-inference-results=100
+
+# Specify a score threshold to be exceeded before program exits with error.
+fail-under=10.0
+
+# Return non-zero exit code if any of these messages/categories are detected,
+# even if score is above --fail-under value. Syntax same as enable. Messages
+# specified are enabled, while categories only check already-enabled messages.
+fail-on=
+
+
+[MESSAGES CONTROL]
+
+# Only show warnings with the listed confidence levels. Leave empty to show
+# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
+# confidence=
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once). See also the "--disable" option for examples.
+enable=
+    use-symbolic-message-instead,
+    useless-suppression,
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once).You can also use "--disable=all" to
+# disable everything first and then re-enable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use"--disable=all --enable=classes
+# --disable=W"
+
+disable=
+    attribute-defined-outside-init,
+    invalid-name,
+    missing-docstring,
+    protected-access,
+    too-few-public-methods,
+    # handled by black
+    format,
+    # We anticipate #3512 where it will become optional
+    fixme,
+    cyclic-import,
+    import-error,
+    # 
+    unnecessary-pass,
+    unrecognized-option,
+    cell-var-from-loop,
+    no-member,
+    wrong-import-order,
+    raise-missing-from,
+    consider-using-f-string
+
+
+[REPORTS]
+
+# Set the output format. Available formats are text, parseable, colorized, msvs
+# (visual studio) and html. You can also give a reporter class, eg
+# mypackage.mymodule.MyReporterClass.
+output-format=text
+
+# Tells whether to display a full report or only the messages
+reports=no
+
+# Python expression which should return a note less than 10 (10 is the highest
+# note). You have access to the variables 'fatal', 'error', 'warning', 'refactor', 'convention'
+# and 'info', which contain the number of messages in each category, as
+# well as 'statement', which is the total number of statements analyzed. This
+# score is used by the global evaluation report (RP0004).
+evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10))
+
+# Template used to display messages. This is a python new-style format string
+# used to format the message information. See doc for all details
+#msg-template=
+
+# Activate the evaluation score.
+score=yes
+
+
+[LOGGING]
+
+# Logging modules to check that the string format arguments are in logging
+# function parameter format
+logging-modules=logging
+
+# The type of string formatting that logging methods do. `old` means using %
+# formatting, `new` is for `{}` formatting.
+logging-format-style=old
+
+
+[MISCELLANEOUS]
+
+# List of note tags to take in consideration, separated by a comma.
+notes=FIXME,XXX,TODO
+
+# Regular expression of note tags to take in consideration.
+#notes-rgx=
+
+
+[SIMILARITIES]
+
+# Minimum lines number of a similarity.
+min-similarity-lines=6
+
+# Ignore comments when computing similarities.
+ignore-comments=yes
+
+# Ignore docstrings when computing similarities.
+ignore-docstrings=yes
+
+# Ignore imports when computing similarities.
+ignore-imports=yes
+
+# Signatures are removed from the similarity computation
+ignore-signatures=yes
+
+
+[VARIABLES]
+
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
+
+# A regular expression matching the name of dummy variables (i.e. expectedly
+# not used).
+dummy-variables-rgx=_$|dummy
+
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid defining new builtins when possible.
+additional-builtins=
+
+# List of strings which can identify a callback function by name. A callback
+# name must start or end with one of those strings.
+callbacks=cb_,_cb
+
+# Tells whether unused global variables should be treated as a violation.
+allow-global-unused-variables=yes
+
+# List of names allowed to shadow builtins
+allowed-redefined-builtins=
+
+# Argument names that match this expression will be ignored. Default to name
+# with leading underscore.
+ignored-argument-names=_.*
+
+# List of qualified module names which can have objects that can redefine
+# builtins.
+redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io
+
+
+[FORMAT]
+
+# Maximum number of characters on a single line.
+max-line-length=120
+
+# Regexp for a line that is allowed to be longer than the limit.
+ignore-long-lines=^\s*(# )?<?https?://\S+>?$
+
+# Allow the body of an if to be on the same line as the test if there is no
+# else.
+single-line-if-stmt=no
+
+# Allow the body of a class to be on the same line as the declaration if body
+# contains single statement.
+single-line-class-stmt=no
+
+# Maximum number of lines in a module
+max-module-lines=1000
+
+# String used as indentation unit. This is usually "    " (4 spaces) or "\t" (1
+# tab).
+indent-string='    '
+
+# Number of spaces of indent required inside a hanging or continued line.
+indent-after-paren=4
+
+# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
+expected-line-ending-format=
+
+
+[BASIC]
+
+# Good variable names which should always be accepted, separated by a comma
+good-names=i,j,k,ex,Run,_
+
+# Good variable names regexes, separated by a comma. If names match any regex,
+# they will always be accepted
+good-names-rgxs=
+
+# Bad variable names which should always be refused, separated by a comma
+bad-names=foo,bar,baz,toto,tutu,tata
+
+# Bad variable names regexes, separated by a comma. If names match any regex,
+# they will always be refused
+bad-names-rgxs=
+
+# Colon-delimited sets of names that determine each other's naming style when
+# the name regexes allow several styles.
+name-group=
+
+# Include a hint for the correct naming format with invalid-name
+include-naming-hint=no
+
+# Naming style matching correct function names.
+function-naming-style=snake_case
+
+# Regular expression matching correct function names
+function-rgx=[a-z_][a-z0-9_]{2,30}$
+
+# Naming style matching correct variable names.
+variable-naming-style=snake_case
+
+# Regular expression matching correct variable names
+variable-rgx=[a-z_][a-z0-9_]{2,30}$
+
+# Naming style matching correct constant names.
+const-naming-style=UPPER_CASE
+
+# Regular expression matching correct constant names
+const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$
+
+# Naming style matching correct attribute names.
+attr-naming-style=snake_case
+
+# Regular expression matching correct attribute names
+attr-rgx=[a-z_][a-z0-9_]{2,}$
+
+# Naming style matching correct argument names.
+argument-naming-style=snake_case
+
+# Regular expression matching correct argument names
+argument-rgx=[a-z_][a-z0-9_]{2,30}$
+
+# Naming style matching correct class attribute names.
+class-attribute-naming-style=any
+
+# Regular expression matching correct class attribute names
+class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
+
+# Naming style matching correct class constant names.
+class-const-naming-style=UPPER_CASE
+
+# Regular expression matching correct class constant names. Overrides class-
+# const-naming-style.
+#class-const-rgx=
+
+# Naming style matching correct inline iteration names.
+inlinevar-naming-style=any
+
+# Regular expression matching correct inline iteration names
+inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
+
+# Naming style matching correct class names.
+class-naming-style=PascalCase
+
+# Regular expression matching correct class names
+class-rgx=[A-Z_][a-zA-Z0-9]+$
+
+
+# Naming style matching correct module names.
+module-naming-style=snake_case
+
+# Regular expression matching correct module names
+module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
+
+
+# Naming style matching correct method names.
+method-naming-style=snake_case
+
+# Regular expression matching correct method names
+method-rgx=[a-z_][a-z0-9_]{2,}$
+
+# Regular expression which can overwrite the naming style set by typevar-naming-style.
+#typevar-rgx=
+
+# Regular expression which should only match function or class names that do
+# not require a docstring. Use ^(?!__init__$)_ to also check __init__.
+no-docstring-rgx=__.*__
+
+# Minimum line length for functions/classes that require docstrings, shorter
+# ones are exempt.
+docstring-min-length=-1
+
+# List of decorators that define properties, such as abc.abstractproperty.
+property-classes=abc.abstractproperty
+
+
+[TYPECHECK]
+
+# Regex pattern to define which classes are considered mixins if ignore-mixin-
+# members is set to 'yes'
+mixin-class-rgx=.*MixIn
+
+# List of module names for which member attributes should not be checked
+# (useful for modules/projects where namespaces are manipulated during runtime
+# and thus existing member attributes cannot be deduced by static analysis). It
+# supports qualified module names, as well as Unix pattern matching.
+ignored-modules=
+
+# List of class names for which member attributes should not be checked (useful
+# for classes with dynamically set attributes). This supports the use of
+# qualified names.
+ignored-classes=SQLObject, optparse.Values, thread._local, _thread._local
+
+# List of members which are set dynamically and missed by pylint inference
+# system, and so shouldn't trigger E1101 when accessed. Python regular
+# expressions are accepted.
+generated-members=REQUEST,acl_users,aq_parent,argparse.Namespace
+
+# List of decorators that create context managers from functions, such as
+# contextlib.contextmanager.
+contextmanager-decorators=contextlib.contextmanager
+
+# Tells whether to warn about missing members when the owner of the attribute
+# is inferred to be None.
+ignore-none=yes
+
+# This flag controls whether pylint should warn about no-member and similar
+# checks whenever an opaque object is returned when inferring. The inference
+# can return multiple potential results while evaluating a Python object, but
+# some branches might not be evaluated, which results in partial inference. In
+# that case, it might be useful to still emit no-member and other checks for
+# the rest of the inferred objects.
+ignore-on-opaque-inference=yes
+
+# Show a hint with possible names when a member name was not found. The aspect
+# of finding the hint is based on edit distance.
+missing-member-hint=yes
+
+# The minimum edit distance a name should have in order to be considered a
+# similar match for a missing member name.
+missing-member-hint-distance=1
+
+# The total number of similar names that should be taken in consideration when
+# showing a hint for a missing member.
+missing-member-max-choices=1
+
+[SPELLING]
+
+# Spelling dictionary name. Available dictionaries: none. To make it working
+# install python-enchant package.
+spelling-dict=
+
+# List of comma separated words that should not be checked.
+spelling-ignore-words=
+
+# List of comma separated words that should be considered directives if they
+# appear and the beginning of a comment and should not be checked.
+spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy:,pragma:,# noinspection
+
+# A path to a file that contains private dictionary; one word per line.
+spelling-private-dict-file=.pyenchant_pylint_custom_dict.txt
+
+# Tells whether to store unknown words to indicated private dictionary in
+# --spelling-private-dict-file option instead of raising a message.
+spelling-store-unknown-words=no
+
+# Limits count of emitted suggestions for spelling mistakes.
+max-spelling-suggestions=2
+
+
+[DESIGN]
+
+# Maximum number of arguments for function / method
+max-args=10
+
+# Maximum number of locals for function / method body
+max-locals=25
+
+# Maximum number of return / yield for function / method body
+max-returns=11
+
+# Maximum number of branch for function / method body
+max-branches=27
+
+# Maximum number of statements in function / method body
+max-statements=100
+
+# Maximum number of parents for a class (see R0901).
+max-parents=7
+
+# List of qualified class names to ignore when counting class parents (see R0901).
+ignored-parents=
+
+# Maximum number of attributes for a class (see R0902).
+max-attributes=11
+
+# Minimum number of public methods for a class (see R0903).
+min-public-methods=2
+
+# Maximum number of public methods for a class (see R0904).
+max-public-methods=25
+
+# Maximum number of boolean expressions in an if statement (see R0916).
+max-bool-expr=5
+
+# List of regular expressions of class ancestor names to
+# ignore when counting public methods (see R0903).
+exclude-too-few-public-methods=
+
+max-complexity=10
+
+[CLASSES]
+
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,__new__,setUp,__post_init__
+
+# List of valid names for the first argument in a class method.
+valid-classmethod-first-arg=cls
+
+# List of valid names for the first argument in a metaclass class method.
+valid-metaclass-classmethod-first-arg=mcs
+
+# List of member names, which should be excluded from the protected access
+# warning.
+exclude-protected=_asdict,_fields,_replace,_source,_make
+
+# Warn about protected attribute access inside special methods
+check-protected-access-in-special-methods=no
+
+[IMPORTS]
+
+# List of modules that can be imported at any level, not just the top level
+# one.
+allow-any-import-level=
+
+# Allow wildcard imports from modules that define __all__.
+allow-wildcard-with-all=no
+
+# Analyse import fallback blocks. This can be used to support both Python 2 and
+# 3 compatible code, which means that the block might have code that exists
+# only in one or another interpreter, leading to false positives when analysed.
+analyse-fallback-blocks=no
+
+# Deprecated modules which should not be used, separated by a comma
+deprecated-modules=regsub,TERMIOS,Bastion,rexec
+
+# Create a graph of every (i.e. internal and external) dependencies in the
+# given file (report RP0402 must not be disabled)
+import-graph=
+
+# Create a graph of external dependencies in the given file (report RP0402 must
+# not be disabled)
+ext-import-graph=
+
+# Create a graph of internal dependencies in the given file (report RP0402 must
+# not be disabled)
+int-import-graph=
+
+# Force import order to recognize a module as part of the standard
+# compatibility libraries.
+known-standard-library=
+
+# Force import order to recognize a module as part of a third party library.
+known-third-party=enchant
+
+# Couples of modules and preferred modules, separated by a comma.
+preferred-modules=
+
+
+[EXCEPTIONS]
+
+# Exceptions that will emit a warning when being caught. Defaults to
+# "Exception"
+overgeneral-exceptions=Exception
+
+
+[TYPING]
+
+# Set to ``no`` if the app / library does **NOT** need to support runtime
+# introspection of type annotations. If you use type annotations
+# **exclusively** for type checking of an application, you're probably fine.
+# For libraries, evaluate if some users what to access the type hints at
+# runtime first, e.g., through ``typing.get_type_hints``. Applies to Python
+# versions 3.7 - 3.9
+runtime-typing = no
+
+
+[DEPRECATED_BUILTINS]
+
+# List of builtins function names that should not be used, separated by a comma
+bad-functions=map,input
+
+
+[REFACTORING]
+
+# Maximum number of nested blocks for function / method body
+max-nested-blocks=5
+
+# Complete name of functions that never returns. When checking for
+# inconsistent-return-statements if a never returning function is called then
+# it will be considered as an explicit return statement and no message will be
+# printed.
+never-returning-functions=sys.exit,argparse.parse_error
+
+
+[STRING]
+
+# This flag controls whether inconsistent-quotes generates a warning when the
+# character used as a quote delimiter is used inconsistently within a module.
+check-quote-consistency=no
+
+# This flag controls whether the implicit-str-concat should generate a warning
+# on implicit string concatenation in sequences defined over several lines.
+check-str-concat-over-line-jumps=no
+
+
+[CODE_STYLE]
+
+# Max line length for which to sill emit suggestions. Used to prevent optional
+# suggestions which would get split by a code formatter (e.g., black). Will
+# default to the setting for ``max-line-length``.
+#max-line-length-suggestions=
+
+W0107:unnecessary-pass

Dockerfile

@@ -0,0 +1,34 @@
+# Use Python 3.10.9 as the base image for consistent runtime environment
+FROM python:3.10.9
+
+# Add metadata labels
+LABEL maintainer="[email protected]"
+LABEL description="Docsifer: Efficient Data Conversion to Markdown using FastAPI and Hugging Face Transformers."
+LABEL version="1.0"
+
+# Setup non-root user for security
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME=/home/user \
+  PATH=/home/user/.local/bin:$PATH
+
+# Set working directory for all subsequent commands
+WORKDIR $HOME/app
+
+# Copy application files
+# Copy requirements first to leverage Docker cache
+COPY --chown=user requirements.txt .
+COPY --chown=user . .
+
+# Install Python dependencies
+# --no-cache-dir reduces image size
+# --upgrade ensures latest compatible versions
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+# Expose service port
+EXPOSE 7860
+
+# Launch FastAPI application using uvicorn server
+# --host 0.0.0.0: Listen on all network interfaces
+# --port 7860: Run on port 7860
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Hieu Lam
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,128 @@
+---
+title: Docsifer
+emoji: 👻 / 📚
+colorFrom: green
+colorTo: indigo
+sdk: docker
+app_file: app.py
+pinned: false
+---
+
+# 📄 Docsifer: Efficient Data Conversion to Markdown
+
+**Docsifer** is a powerful FastAPI + Gradio service for converting various data formats (PDF, PowerPoint, Word, Excel, Images, Audio, HTML, etc.) to Markdown. It leverages the [MarkItDown](https://github.com/microsoft/markitdown) library and can optionally use LLMs (via OpenAI) for richer extraction (OCR, speech-to-text, etc.).
+
+## ✨ Key Features
+
+- **Comprehensive Format Support**: 
+  - **PDF**: Extracts text and structure effectively.
+  - **PowerPoint**: Converts slides into Markdown-friendly content.
+  - **Word**: Processes `.docx` files with precision.
+  - **Excel**: Extracts tabular data as Markdown tables.
+  - **Images**: Reads **EXIF metadata** and applies **OCR** for text extraction.
+  - **Audio**: Retrieves **EXIF metadata** and performs **speech transcription**.
+  - **HTML**: Transforms web pages into Markdown.
+  - **Text-Based Formats**: Handles CSV, JSON, XML with ease.
+  - **ZIP Files**: Iterates over contents for batch processing.
+- **LLM Integration**: Leverages OpenAI's GPT-4 for enhanced extraction quality and contextual understanding.
+- **Efficient and Fast**: Optimized for speed while maintaining high accuracy.
+- **Easy Deployment**: Dockerized for hassle-free setup and scalability.
+- **Interactive Playground**: Test conversion processes interactively using a **Gradio-powered interface**.
+- **Usage Analytics**: Tracks token usage and access statistics via Upstash Redis.
+
+## 🚀 Use Cases
+
+- **Knowledge Indexing**: Convert various document formats into Markdown for indexing and search.
+- **Text Analysis**: Prepare data for semantic analysis and NLP tasks.
+- **Content Transformation**: Simplify content preparation for blogs, documentation, or databases.
+- **Metadata Extraction**: Extract meaningful metadata from images and audio for categorization and tagging.
+
+## 🛠️ Getting Started
+
+### 1. Clone the Repository
+
+```bash
+git clone https://github.com/lh0x00/docsifer.git
+cd docsifer
+```
+
+### 2. Build and Run with Docker
+
+Make sure Docker is installed and running on your machine.
+
+```bash
+docker build -t docsifer .
+docker run -p 7860:7860 docsifer
+```
+
+The API will now be accessible at `http://localhost:7860`.
+
+### 3. Environment Variables
+
+Set the following environment variables as needed:
+
+- `OPENAI_API_KEY`: Your OpenAI API key (optional, for LLM-enhanced features).
+- `OPENAI_BASE_URL`: Custom base endpoint for OpenAI-compatible services (optional).
+- `REDIS_URL`: Upstash Redis URL (default: `redis://localhost:6379/0`).
+- `REDIS_TOKEN`: Upstash Redis token (default: `***`).
+
+You can set these in a `.env` file or pass them directly when running the Docker container:
+
+```bash
+docker run -p 7860:7860 \
+  -e OPENAI_API_KEY=sk-xxxxx \
+  -e OPENAI_BASE_URL=https://api.openai.com/v1 \
+  -e REDIS_URL=redis://your-upstash-url \
+  -e REDIS_TOKEN=your-upstash-token \
+  docsifer
+```
+
+## 📖 API Overview
+
+### Endpoints
+
+- **`/v1/convert`**: Convert a file to Markdown. Supports both file uploads and file path inputs. Accepts optional OpenAI parameters to enable LLM-based enhancements.
+- **`/v1/stats`**: Retrieve usage statistics, including access counts and token usage.
+
+### Interactive Docs
+
+- Visit the [Swagger UI](http://localhost:7860/docs) for detailed, interactive documentation.
+- Explore additional resources with [ReDoc](http://localhost:7860/redoc).
+
+## 🔬 Playground
+
+### Interactive Conversion
+
+- Test file conversion directly in the browser using the **Gradio interface**.
+- Simply visit `http://localhost:7860` after starting the server to access the playground.
+
+### Features
+
+- **File Upload**: Upload a file directly or provide a local file path.
+- **OpenAI Integration**: Optionally provide OpenAI API details to enhance conversion with LLM capabilities.
+- **Conversion Result**: View the resulting Markdown output instantly.
+- **Usage Statistics**: Monitor access and token usage through the Gradio interface.
+
+## 🌐 Resources
+
+- **Documentation**: [Explore full documentation](https://lamhieu-docsifer.hf.space/docs)
+- **Hugging Face Space**: [Try the live demo](https://huggingface.co/spaces/lh0x00/docsifer)
+- **GitHub Repository**: [View source code](https://github.com/lh0x00/docsifer)
+
+## 💡 Why Docsifer?
+
+1. **Versatile and Comprehensive**: Handles a wide range of formats, making it a one-stop solution for content conversion.
+2. **AI-Powered**: Uses OpenAI's GPT-4 to enhance extraction accuracy and adapt to complex data structures.
+3. **User-Friendly**: Offers intuitive APIs and a built-in interactive interface for experimentation.
+4. **Scalable and Efficient**: Optimized for performance with Docker support and asynchronous processing.
+5. **Transparent Analytics**: Tracks usage metrics to help monitor and manage service consumption.
+
+## 👥 Contributors
+
+- **lamhieu / lh0x00** – Creator and Maintainer ([GitHub](https://github.com/lh0x00))
+
+Contributions are welcome! Check out the [contribution guidelines](https://github.com/lh0x00/docsifer/blob/main/CONTRIBUTING.md).
+
+## 📜 License
+
+This project is licensed under the **MIT License**. See the [LICENSE](https://github.com/lh0x00/docsifer/blob/main/LICENSE) file for details.

app.py

@@ -0,0 +1 @@
+from docsifer import app

docsifer/__init__.py

@@ -0,0 +1,384 @@
+# filename: __init__.py
+
+import json
+import logging
+import tempfile
+from typing import Tuple, Optional
+
+import gradio as gr
+import pandas as pd
+import requests
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from gradio.routes import mount_gradio_app
+from pathlib import Path
+
+# If you want to generate unique filenames, e.g. scuid:
+from scuid import scuid
+
+
+# Filter out /v1 requests from the access log
+class LogFilter(logging.Filter):
+    def filter(self, record):
+        # Only keep log records that contain "/v1" in the request path
+        if record.args and len(record.args) >= 3:
+            if "/v1" in str(record.args[2]):
+                return True
+        return False
+
+
+logger = logging.getLogger("uvicorn.access")
+logger.addFilter(LogFilter())
+
+# Application metadata
+__version__ = "1.0.0"
+__author__ = "lamhieu"
+__description__ = "Docsifer: Efficient Data Conversion to Markdown."
+__metadata__ = {
+    "project": "Docsifer",
+    "version": __version__,
+    "description": (
+        "Effortlessly convert various files to Markdown, including PDF, PowerPoint, Word, Excel, "
+        "images, audio, HTML, JSON, CSV, XML, ZIP, and more."
+    ),
+    "docs": "https://lamhieu-docsifer.hf.space/docs",
+    "github": "https://github.com/lh0x00/docsifer",
+    "spaces": "https://huggingface.co/spaces/lh0x00/docsifer",
+}
+
+# Update your Docsifer API endpoints (you can replace with your HF Space or other URL)
+DOCSIFER_API_URL = "http://localhost:7860/v1/convert"
+DOCSIFER_STATS_URL = "http://localhost:7860/v1/stats"
+
+# Markdown description for the main interface
+APP_DESCRIPTION = f"""
+# 📝 **Docsifer: Convert Your Documents to Markdown**
+
+Welcome to **Docsifer**, a specialized service that converts your files—like PDF, PPT, Word, Excel, images, audio, HTML, JSON, CSV, XML, ZIP, etc.—into **Markdown** using **MarkItDown** at the core. Optionally, you can leverage **LLMs** (OpenAI) for advanced text extraction.
+
+### Features & Privacy
+
+- **Open Source**: The entire Docsifer codebase is publicly available for review and contribution.
+- **Efficient & Flexible**: Supports multiple file formats, ensuring quick and accurate Markdown conversion.
+- **Privacy-Focused**: We never store user data; all processing is ephemeral. We only collect minimal anonymous usage stats for service improvement.
+- **Production-Ready**: Easy Docker deployment, interactive Gradio playground, and comprehensive REST API documentation.
+- **Community & Collaboration**: Contribute on [GitHub]({__metadata__["github"]}) or try it out on [Hugging Face Spaces]({__metadata__["spaces"]}).
+
+### 🔗 Resources
+- [Documentation]({__metadata__["docs"]}) | [GitHub]({__metadata__["github"]}) | [Live Demo]({__metadata__["spaces"]})
+"""
+
+# Initialize FastAPI application
+app = FastAPI(
+    title="Docsifer Service API",
+    description=__description__,
+    version=__version__,
+    docs_url="/docs",
+    redoc_url="/redoc",
+)
+
+# Configure CORS
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],  # Adjust if needed for specific domains
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Import and include your existing router (which has /v1/convert, /v1/stats, etc.)
+from .router import router
+
+app.include_router(router, prefix="/v1")
+
+
+def call_convert_api(
+    file_obj: bytes,
+    filename: str,
+    cleanup: bool = True,
+    openai_base_url: Optional[str] = None,
+    openai_api_key: Optional[str] = None,
+    openai_model: Optional[str] = None,
+) -> Tuple[str, str]:
+    """
+    Calls the /v1/convert endpoint, returning (markdown_content, md_file_path).
+    If there's an error, the first return value is an error message (str),
+    the second is an empty string.
+
+    The updated /v1/convert expects:
+      - file (UploadFile)
+      - openai (object, e.g. {"api_key":"...","base_url":"..."})
+      - settings (object, e.g. {"cleanup": true})
+    """
+
+    if file_obj is None:
+        return ("❌ No file was uploaded.", "")
+
+    # Build the "openai" object
+    openai_dict = {}
+    if openai_api_key and openai_api_key.strip():
+        openai_dict["api_key"] = openai_api_key
+    if openai_base_url and openai_base_url.strip():
+        openai_dict["base_url"] = openai_base_url
+    if openai_model and openai_model.strip():
+        openai_dict["model"] = openai_model
+
+    # Build the "settings" object
+    settings_dict = {"cleanup": cleanup}
+
+    data = {
+        # These must match the `Form(...)` fields named "openai" and "settings"
+        "openai": json.dumps(openai_dict),
+        "settings": json.dumps(settings_dict),
+    }
+
+    if len(openai_dict) <= 3:
+        data.pop("openai")
+
+    # Prepare files for multipart/form-data
+    files = {"file": (filename, file_obj)}
+
+    try:
+        response = requests.post(DOCSIFER_API_URL, files=files, data=data, timeout=30)
+    except requests.exceptions.RequestException as e:
+        return (f"❌ Network Error: {str(e)}", "")
+
+    if response.status_code != 200:
+        return (f"❌ API Error {response.status_code}: {response.text}", "")
+
+    try:
+        converted = response.json()
+        # Expecting { "filename": "...", "markdown": "..." }
+        markdown_content = converted["markdown"]
+    except Exception as e:
+        return (f"❌ Error parsing JSON: {str(e)}", "")
+
+    # Write the returned Markdown to a temporary .md file so Gradio can serve it
+    with tempfile.NamedTemporaryFile(
+        mode="w+", suffix=".md", dir="/tmp", delete=False
+    ) as tmp_file:
+        tmp_file.write(markdown_content)
+        tmp_md_path = tmp_file.name
+
+    return (markdown_content, tmp_md_path)
+
+
+def call_stats_api_df() -> Tuple[pd.DataFrame, pd.DataFrame]:
+    """
+    Calls /v1/stats endpoint to retrieve analytics data.
+    Returns two DataFrames: (access_df, tokens_df).
+    """
+    try:
+        response = requests.get(DOCSIFER_STATS_URL, timeout=10)
+    except requests.exceptions.RequestException as e:
+        raise ValueError(f"Failed to fetch stats: {str(e)}")
+
+    if response.status_code != 200:
+        raise ValueError(f"Failed to fetch stats: {response.text}")
+
+    data = response.json()
+    # Expected structure:
+    # {
+    #   "access": { <period>: {"docsifer": count, ...}, ... },
+    #   "tokens": { <period>: {"docsifer": count, ...}, ... }
+    # }
+    access_data = data.get("access", {})
+    tokens_data = data.get("tokens", {})
+
+    def build_stats_df(bucket: dict) -> pd.DataFrame:
+        # We want columns for periods: total, daily, weekly, monthly, yearly
+        # Each row => "docsifer" (just 1 row if everything is aggregated)
+        all_models = set()
+        for period_key in ["total", "daily", "weekly", "monthly", "yearly"]:
+            period_dict = bucket.get(period_key, {})
+            all_models.update(period_dict.keys())  # typically just "docsifer"
+
+        result_dict = {
+            "Model": [],
+            "Total": [],
+            "Daily": [],
+            "Weekly": [],
+            "Monthly": [],
+            "Yearly": [],
+        }
+
+        for model in sorted(all_models):
+            result_dict["Model"].append(model)
+            result_dict["Total"].append(bucket.get("total", {}).get(model, 0))
+            result_dict["Daily"].append(bucket.get("daily", {}).get(model, 0))
+            result_dict["Weekly"].append(bucket.get("weekly", {}).get(model, 0))
+            result_dict["Monthly"].append(bucket.get("monthly", {}).get(model, 0))
+            result_dict["Yearly"].append(bucket.get("yearly", {}).get(model, 0))
+
+        return pd.DataFrame(result_dict)
+
+    access_df = build_stats_df(access_data)
+    tokens_df = build_stats_df(tokens_data)
+    return access_df, tokens_df
+
+
+def create_main_interface():
+    """
+    Creates a Gradio Blocks interface:
+    - A 'Conversion Playground' tab for uploading a file and converting to Markdown
+    - An 'Analytics Stats' section to display usage statistics
+    - cURL examples for reference
+    """
+    with gr.Blocks(title="Docsifer: Convert to Markdown", theme="default") as demo:
+        gr.Markdown(APP_DESCRIPTION)
+
+        with gr.Tab("Conversion Playground"):
+            gr.Markdown("### Convert your files to Markdown with Docsifer.")
+
+            with gr.Row():
+                with gr.Column():
+                    file_input = gr.File(
+                        label="Upload File",
+                        file_types=[
+                            ".pdf",
+                            ".docx",
+                            ".pptx",
+                            ".xlsx",
+                            ".html",
+                            ".htm",
+                            ".jpg",
+                            ".jpeg",
+                            ".png",
+                            ".mp3",
+                            ".wav",
+                            ".zip",
+                        ],
+                        type="binary",
+                    )
+
+                    with gr.Accordion("OpenAI Configuration (Optional)", open=False):
+                        gr.Markdown(
+                            "Provide these if you'd like **LLM-assisted** extraction. "
+                            "Supports both OpenAI and OpenAI-compatible APIs. "
+                            "If left blank, basic conversion (no LLM) will be used."
+                        )
+                        openai_base_url = gr.Textbox(
+                            label="Base URL",
+                            placeholder="https://api.openai.com/v1",
+                            value="https://api.openai.com/v1",
+                        )
+                        openai_api_key = gr.Textbox(
+                            label="API Key",
+                            placeholder="sk-...",
+                            type="password",
+                        )
+                        openai_model = gr.Textbox(
+                            label="Model ID",
+                            placeholder="e.g. gpt-4o-mini",
+                            value="gpt-4o-mini",
+                        )
+
+                    with gr.Accordion("Conversion Settings", open=True):
+                        gr.Markdown(
+                            "Enable to remove <style> tags or hidden elements from `.html` files before conversion."
+                        )
+                        cleanup_toggle = gr.Checkbox(
+                            label="Enable Cleanup",
+                            value=True,
+                        )
+
+                    convert_btn = gr.Button("Convert")
+
+                with gr.Column():
+                    output_md = gr.Textbox(
+                        label="Conversion Result (Markdown)",
+                        lines=20,
+                        interactive=False,
+                    )
+                    # Set visible=True so the user always sees a small download button
+                    download_file = gr.File(
+                        label="Download",
+                        interactive=False,
+                        visible=True,
+                    )
+
+                    gr.Markdown(
+                        """
+                        ### cURL Examples
+
+                        **Convert via File Upload (multipart/form-data)**:
+                        ```bash
+                        curl -X POST \\
+                            "https://lamhieu-docsifer.hf.space/v1/convert" \\
+                            -F "file=@/path/to/local/document.pdf" \\
+                            -F "openai={{\\"api_key\\":\\"sk-xxxxx\\",\\"model\\":\\"gpt-4o-mini\\",\\"base_url\\":\\"https://api.openai.com/v1\\"}}" \\
+                            -F "settings={{\\"cleanup\\":true}}"
+                        ```
+                        """
+                    )
+
+            def on_convert(file_bytes, base_url, api_key, model_id, cleanup):
+                """
+                Callback for the 'Convert' button.
+                We generate a unique name if the user uploads a file.
+                """
+                if not file_bytes:
+                    return "❌ Please upload a file first.", None
+
+                unique_name = f"{scuid()}.tmp"
+                markdown, temp_md_path = call_convert_api(
+                    file_obj=file_bytes,
+                    filename=unique_name,
+                    openai_base_url=base_url,
+                    openai_api_key=api_key,
+                    openai_model=model_id,
+                    cleanup=cleanup,
+                )
+                return markdown, temp_md_path
+
+            convert_btn.click(
+                fn=on_convert,
+                inputs=[
+                    file_input,
+                    openai_base_url,
+                    openai_api_key,
+                    openai_model,
+                    cleanup_toggle,
+                ],
+                outputs=[output_md, download_file],
+            )
+
+        with gr.Tab("Analytics Stats"):
+            gr.Markdown(
+                "View Docsifer usage statistics (access count, token usage, etc.)"
+            )
+            stats_btn = gr.Button("Get Stats")
+            access_df = gr.DataFrame(
+                label="Access Stats",
+                headers=["Model", "Total", "Daily", "Weekly", "Monthly", "Yearly"],
+                interactive=False,
+            )
+            tokens_df = gr.DataFrame(
+                label="Token Stats",
+                headers=["Model", "Total", "Daily", "Weekly", "Monthly", "Yearly"],
+                interactive=False,
+            )
+
+            stats_btn.click(
+                fn=call_stats_api_df,
+                inputs=[],
+                outputs=[access_df, tokens_df],
+            )
+
+    return demo
+
+
+# Build our Gradio interface and mount it at the root path
+main_interface = create_main_interface()
+mount_gradio_app(app, main_interface, path="/")
+
+
+# Startup / Shutdown events
+@app.on_event("startup")
+async def startup_event():
+    logger.info("Docsifer Service is starting up...")
+
+
+@app.on_event("shutdown")
+async def shutdown_event():
+    logger.info("Docsifer Service is shutting down.")

docsifer/analytics.py

@@ -0,0 +1,290 @@
+# filename: analytics.py
+
+import logging
+import asyncio
+from upstash_redis import Redis as UpstashRedis
+from datetime import datetime
+from collections import defaultdict
+from typing import Dict
+from functools import partial
+
+logger = logging.getLogger(__name__)
+
+
+class Analytics:
+    def __init__(
+        self, url: str, token: str, sync_interval: int = 60, max_retries: int = 5
+    ):
+        """
+        Initializes the Analytics class with an Upstash Redis client (HTTP-based),
+        wrapped in async methods by using run_in_executor.
+
+        We maintain two dictionaries:
+         - current_totals: absolute counters (loaded from Redis at startup).
+         - new_increments: only the new usage since last sync.
+
+        Both structures only track a single label "docsifer" for access/tokens.
+        """
+        self.url = url
+        self.token = token
+        self.sync_interval = sync_interval
+        self.max_retries = max_retries
+
+        # Create the synchronous Upstash Redis client over HTTP
+        self.redis_client = self._create_redis_client()
+
+        # current_totals: absolute counters from Redis
+        self.current_totals = {
+            "access": defaultdict(lambda: defaultdict(int)),
+            "tokens": defaultdict(lambda: defaultdict(int)),
+        }
+        # new_increments: only new usage since the last successful sync
+        self.new_increments = {
+            "access": defaultdict(lambda: defaultdict(int)),
+            "tokens": defaultdict(lambda: defaultdict(int)),
+        }
+
+        # Async lock to protect shared data
+        self.lock = asyncio.Lock()
+
+        # Start initial sync from Redis, then a periodic sync task
+        asyncio.create_task(self._initialize())
+
+        logger.info("Initialized Analytics with Upstash Redis: %s", url)
+
+    def _create_redis_client(self) -> UpstashRedis:
+        """Creates and returns a new Upstash Redis (synchronous) client."""
+        return UpstashRedis(url=self.url, token=self.token)
+
+    async def _initialize(self):
+        """
+        Fetch existing data from Redis into current_totals,
+        then start the periodic sync task.
+        """
+        try:
+            await self._sync_from_redis()
+            logger.info("Initial sync from Upstash Redis completed successfully.")
+        except Exception as e:
+            logger.error("Error during initial Redis sync: %s", e)
+
+        asyncio.create_task(self._start_sync_task())
+
+    def _get_period_keys(self):
+        """
+        Returns day, week, month, year keys based on current UTC date.
+        Also consider "total" if you want an all-time key.
+
+        Example: ("2025-01-14", "2025-W02", "2025-01", "2025", "total")
+        """
+        now = datetime.utcnow()
+        day_key = now.strftime("%Y-%m-%d")
+        week_key = f"{now.year}-W{now.strftime('%U')}"
+        month_key = now.strftime("%Y-%m")
+        year_key = now.strftime("%Y")
+        # For convenience, also track everything in "total".
+        return day_key, week_key, month_key, year_key, "total"
+
+    async def access(self, tokens: int):
+        """
+        Records an access and token usage for the "docsifer" label.
+        This function updates both current_totals and new_increments.
+        """
+        day_key, week_key, month_key, year_key, total_key = self._get_period_keys()
+
+        async with self.lock:
+            # For each time period, increment "docsifer" usage
+            for period in [day_key, week_key, month_key, year_key, total_key]:
+                # Increase new usage
+                self.new_increments["access"][period]["docsifer"] += 1
+                self.new_increments["tokens"][period]["docsifer"] += tokens
+
+                # Also update the absolute totals for immediate stats
+                self.current_totals["access"][period]["docsifer"] += 1
+                self.current_totals["tokens"][period]["docsifer"] += tokens
+
+    async def stats(self) -> Dict[str, Dict[str, Dict[str, int]]]:
+        """
+        Returns a snapshot of current stats (absolute totals).
+        We use current_totals, which is always up to date.
+        """
+        async with self.lock:
+            return {
+                "access": {
+                    period: dict(models)
+                    for period, models in self.current_totals["access"].items()
+                },
+                "tokens": {
+                    period: dict(models)
+                    for period, models in self.current_totals["tokens"].items()
+                },
+            }
+
+    async def _sync_from_redis(self):
+        """
+        Pull existing data from Redis into current_totals and reset new_increments.
+        We read "analytics:access:*" and "analytics:tokens:*" keys via SCAN.
+        """
+        loop = asyncio.get_running_loop()
+
+        async with self.lock:
+            # Reset both structures
+            self.current_totals = {
+                "access": defaultdict(lambda: defaultdict(int)),
+                "tokens": defaultdict(lambda: defaultdict(int)),
+            }
+            self.new_increments = {
+                "access": defaultdict(lambda: defaultdict(int)),
+                "tokens": defaultdict(lambda: defaultdict(int)),
+            }
+
+            # ---------------------------
+            # Load "access" data
+            # ---------------------------
+            cursor = 0
+            while True:
+                scan_result = await loop.run_in_executor(
+                    None,
+                    partial(
+                        self.redis_client.scan,
+                        cursor=cursor,
+                        match="analytics:access:*",
+                        count=1000,
+                    ),
+                )
+                cursor, keys = scan_result[0], scan_result[1]
+
+                for key in keys:
+                    # key => "analytics:access:<period>"
+                    period = key.replace("analytics:access:", "")
+                    data = await loop.run_in_executor(
+                        None,
+                        partial(self.redis_client.hgetall, key),
+                    )
+                    for name_key, count_str in data.items():
+                        self.current_totals["access"][period][name_key] = int(count_str)
+
+                if cursor == 0:
+                    break
+
+            # ---------------------------
+            # Load "tokens" data
+            # ---------------------------
+            cursor = 0
+            while True:
+                scan_result = await loop.run_in_executor(
+                    None,
+                    partial(
+                        self.redis_client.scan,
+                        cursor=cursor,
+                        match="analytics:tokens:*",
+                        count=1000,
+                    ),
+                )
+                cursor, keys = scan_result[0], scan_result[1]
+
+                for key in keys:
+                    # key => "analytics:tokens:<period>"
+                    period = key.replace("analytics:tokens:", "")
+                    data = await loop.run_in_executor(
+                        None,
+                        partial(self.redis_client.hgetall, key),
+                    )
+                    for name_key, count_str in data.items():
+                        self.current_totals["tokens"][period][name_key] = int(count_str)
+
+                if cursor == 0:
+                    break
+
+    async def _sync_to_redis(self):
+        """
+        Push the new_increments to Redis with HINCRBY,
+        then reset new_increments to zero if successful.
+        """
+        loop = asyncio.get_running_loop()
+
+        async with self.lock:
+            try:
+                # Sync "access" increments
+                for period, models in self.new_increments["access"].items():
+                    redis_key = f"analytics:access:{period}"
+                    for name_key, count_val in models.items():
+                        if count_val != 0:
+                            await loop.run_in_executor(
+                                None,
+                                partial(
+                                    self.redis_client.hincrby,
+                                    redis_key,
+                                    name_key,
+                                    count_val,
+                                ),
+                            )
+
+                # Sync "tokens" increments
+                for period, models in self.new_increments["tokens"].items():
+                    redis_key = f"analytics:tokens:{period}"
+                    for name_key, count_val in models.items():
+                        if count_val != 0:
+                            await loop.run_in_executor(
+                                None,
+                                partial(
+                                    self.redis_client.hincrby,
+                                    redis_key,
+                                    name_key,
+                                    count_val,
+                                ),
+                            )
+
+                logger.info("Analytics data synced to Upstash Redis.")
+
+                # Reset new_increments only
+                self.new_increments = {
+                    "access": defaultdict(lambda: defaultdict(int)),
+                    "tokens": defaultdict(lambda: defaultdict(int)),
+                }
+
+            except Exception as e:
+                logger.error("Error syncing to Redis: %s", e)
+                raise e
+
+    async def _start_sync_task(self):
+        """Periodically sync local increments to Redis."""
+        while True:
+            await asyncio.sleep(self.sync_interval)
+            try:
+                await self._sync_to_redis()
+            except Exception as e:
+                logger.error("Error during scheduled sync: %s", e)
+                await self._handle_redis_reconnection()
+
+    async def _handle_redis_reconnection(self):
+        """
+        Attempts to reconnect to Redis if connection fails (HTTP-based, stateless).
+        """
+        loop = asyncio.get_running_loop()
+        retry_count = 0
+        delay = 1
+
+        while retry_count < self.max_retries:
+            try:
+                logger.info(
+                    "Attempting Redis reconnection (attempt %d)...", retry_count + 1
+                )
+                await loop.run_in_executor(None, self.redis_client.close)
+                self.redis_client = self._create_redis_client()
+                logger.info("Reconnected to Redis successfully.")
+                return
+            except Exception as e:
+                logger.error("Reconnection attempt %d failed: %s", retry_count + 1, e)
+                retry_count += 1
+                await asyncio.sleep(delay)
+                delay *= 2
+
+        logger.critical("Max reconnection attempts reached. Redis is unavailable.")
+
+    async def close(self):
+        """
+        Close the Upstash Redis client (though it's stateless over HTTP).
+        """
+        loop = asyncio.get_running_loop()
+        await loop.run_in_executor(None, self.redis_client.close)
+        logger.info("Redis client closed.")

docsifer/router.py

@@ -0,0 +1,92 @@
+# filename: router.py
+
+import logging
+import json
+import tempfile
+import os
+from pathlib import Path
+
+from fastapi import APIRouter, HTTPException, UploadFile, File, Form, BackgroundTasks
+from pydantic import BaseModel
+
+from .service import DocsiferService
+from .analytics import Analytics
+
+logger = logging.getLogger(__name__)
+router = APIRouter(tags=["v1"], responses={404: {"description": "Not found"}})
+
+# Initialize analytics (single aggregator = "docsifer")
+analytics = Analytics(
+    url=os.environ.get("REDIS_URL", "redis://localhost:6379/0"),
+    token=os.environ.get("REDIS_TOKEN", "***"),
+    sync_interval=30 * 60,  # e.g. 30 minutes
+)
+
+# Initialize the Docsifer service (token counting with gpt-4o)
+docsifer_service = DocsiferService(model_name="gpt-4o")
+
+
+class ConvertResponse(BaseModel):
+    filename: str
+    markdown: str
+
+
+@router.post("/convert", response_model=ConvertResponse)
+async def convert_document(
+    background_tasks: BackgroundTasks,
+    file: UploadFile = File(..., description="File to convert (1 file per request)"),
+    openai: str = Form("{}", description="OpenAI config as a JSON object"),
+    settings: str = Form("{}", description="Settings as a JSON object"),
+):
+    """
+    Convert a single uploaded file to Markdown, optionally using OpenAI for advanced text extraction.
+    - `openai` is a JSON string with keys: {"api_key": "...", "base_url": "..."}
+    - `settings` is a JSON string with keys: {"cleanup": bool}
+    - We do not store or track model_id in analytics; everything is aggregated as "docsifer".
+    """
+    try:
+        try:
+            openai_config = json.loads(openai) if openai else {}
+        except json.JSONDecodeError:
+            raise ValueError("Invalid JSON in 'openai' parameter.")
+
+        try:
+            settings_config = json.loads(settings) if settings else {}
+        except json.JSONDecodeError:
+            raise ValueError("Invalid JSON in 'settings' parameter.")
+
+        cleanup = settings_config.get("cleanup", True)
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            temp_path = Path(tmpdir) / file.filename
+            contents = await file.read()
+            temp_path.write_bytes(contents)
+
+            result, token_count = await docsifer_service.convert_file(
+                file_path=str(temp_path), openai_config=openai_config, cleanup=cleanup
+            )
+
+        # Track usage in analytics (single aggregator => "docsifer")
+        background_tasks.add_task(analytics.access, token_count)
+
+        return ConvertResponse(**result)
+
+    except Exception as e:
+        msg = f"Failed to convert document. Error: {str(e)}"
+        logger.error(msg)
+        raise HTTPException(status_code=500, detail=msg)
+
+
+@router.get("/stats")
+async def get_stats():
+    """
+    Return usage statistics (access, tokens) from the Analytics system.
+    All data is stored under "docsifer".
+    """
+    try:
+        data = await analytics.stats()
+        return data
+    except Exception as e:
+        msg = f"Failed to fetch analytics stats: {str(e)}"
+        logger.error(msg)
+        raise HTTPException(status_code=500, detail=msg)

docsifer/service.py

@@ -0,0 +1,133 @@
+# filename: service.py
+
+from __future__ import annotations
+
+import logging
+import tempfile
+from pathlib import Path
+from typing import Optional, Dict, Tuple, Any
+
+import tiktoken
+from pyquery import PyQuery as pq
+from markitdown import MarkItDown
+from openai import OpenAI
+
+
+logger = logging.getLogger(__name__)
+
+
+class DocsiferService:
+    """
+    A service that converts local files to Markdown using MarkItDown,
+    optionally with an OpenAI LLM for advanced extraction.
+    Token counting uses "gpt-4o" as a heuristic via tiktoken.
+    """
+
+    def __init__(self, model_name: str = "gpt-4o"):
+        """
+        Initialize the DocsiferService with a basic MarkItDown instance
+        and a tiktoken encoder for counting tokens using "gpt-4o".
+        """
+        self._basic_markitdown = MarkItDown()  # MarkItDown without LLM
+        # Use "gpt-4o" for token counting
+        try:
+            self._encoder = tiktoken.encoding_for_model(model_name)
+        except Exception as e:
+            logger.warning(
+                "Error loading tiktoken model '%s': %s. Falling back to 'gpt-3.5-turbo-0301'.",
+                model_name,
+                e,
+            )
+            self._encoder = tiktoken.encoding_for_model("gpt-3.5-turbo-0301")
+
+        logger.info("DocsiferService initialized with token model '%s'.", model_name)
+
+    def _init_markitdown_with_llm(self, openai_config: Dict[str, Any]) -> MarkItDown:
+        """
+        If openai_config has an 'api_key', configure openai and return
+        a MarkItDown instance with that OpenAI client.
+        """
+        api_key = openai_config.get("api_key", "")
+        if not api_key:
+            logger.info("No OpenAI API key provided. Using basic MarkItDown.")
+            return self._basic_markitdown
+
+        model = openai_config.get("model", "gpt-4o-mini")
+        base_url = openai_config.get("base_url", "https://api.openai.com/v1")
+        client = OpenAI(api_key=api_key, base_url=base_url)
+
+        logger.info("Initialized OpenAI with base_url=%s", base_url)
+        return MarkItDown(llm_client=client, llm_model=model)
+
+    def _maybe_cleanup_html(self, html_file: Path) -> None:
+        """
+        If the file is HTML, remove <style> tags, optionally hidden elements, etc.
+        """
+        try:
+            content = html_file.read_text(encoding="utf-8", errors="ignore")
+            d = pq(content)
+            # Remove hidden elements
+            d('*[style*="display:none"]').remove()
+            cleaned_html = str(d)
+            cleaned_html = cleaned_html.strip()
+            html_file.write_text(cleaned_html, encoding="utf-8")
+        except Exception as e:
+            logger.error("HTML cleanup failed for %s: %s", html_file, e)
+
+    def _count_tokens(self, text: str) -> int:
+        """
+        Count tokens using the configured tiktoken encoder.
+        Fallback to whitespace-based counting if an error occurs.
+        """
+        try:
+            return len(self._encoder.encode(text))
+        except Exception as e:
+            logger.warning(
+                "Token counting failed, fallback to whitespace. Error: %s", e
+            )
+            return len(text.split())
+
+    async def convert_file(
+        self, file_path: str, openai_config: Optional[dict] = None, cleanup: bool = True
+    ) -> Tuple[Dict[str, str], int]:
+        """
+        Converts a file at `file_path` to Markdown.
+        - If `cleanup` is True and file is .html/.htm, does HTML cleanup.
+        - If `openai_config` has a valid API key, use LLM-based MarkItDown.
+        Returns ({"filename": filename, "markdown": md_string}, token_count).
+        """
+        src = Path(file_path)
+        if not src.exists():
+            raise FileNotFoundError(f"File not found: {file_path}")
+
+        logger.info("Converting file: %s (cleanup=%s)", file_path, cleanup)
+
+        # Use a temp directory so MarkItDown sees the real file extension
+        with tempfile.TemporaryDirectory() as tmpdir:
+            tmp_path = Path(tmpdir) / src.name
+            tmp_path.write_bytes(src.read_bytes())
+
+            # If it's HTML and cleanup is requested
+            if cleanup and tmp_path.suffix.lower() in (".html", ".htm"):
+                self._maybe_cleanup_html(tmp_path)
+
+            # Decide whether to use LLM or basic
+            if openai_config and openai_config.get("api_key"):
+                md_converter = self._init_markitdown_with_llm(openai_config)
+            else:
+                md_converter = self._basic_markitdown
+
+            try:
+                result_obj = md_converter.convert(str(tmp_path))
+            except Exception as e:
+                logger.error("MarkItDown conversion failed: %s", e)
+                raise RuntimeError(f"Conversion failed for '{file_path}': {e}")
+
+            # Count tokens
+            token_count = self._count_tokens(result_obj.text_content)
+
+            result_dict = {
+                "filename": src.name,
+                "markdown": result_obj.text_content,
+            }
+            return result_dict, token_count

pyproject.toml

@@ -0,0 +1,17 @@
+[tool.poetry]
+name = "docsifer"
+version = "1.0.0"
+description = "Docsifer is a powerful tool for converting various data formats into Markdown for applications such as indexing, text analysis, and more. It supports PDF, PowerPoint, Word, Excel, Images, Audio, HTML, and other text-based formats, and leverages Large Language Models (LLMs) to enhance performance."
+authors = ["Hieu Lam <[email protected]>"]
+readme = "README.md"
+homepage = "https://github.com/lh0x00/docsifer"
+repository = "https://github.com/lh0x00/docsifer"
+license = "MIT"
+
+[tool.poetry.dependencies]
+python = "^3.10"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

requirements.txt

@@ -0,0 +1,15 @@
+gradio
+fastapi
+uvicorn
+requests
+pydantic
+cachetools
+upstash_redis==1.2.0
+markitdown
+openai
+pyquery
+tiktoken
+scuid
+python-magic
+plotly
+matplotlib