Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: read Sage Payroll pdf and compare #22

Merged
merged 2 commits into from
Feb 21, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/pr.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ jobs:
--health-retries 5
strategy:
matrix:
python: ["3.10"]
python: ["3.10", "3.12"]
steps:
- name: Setup Python
uses: actions/setup-python@v2
Expand Down
18 changes: 18 additions & 0 deletions django-kitamanager/kitamanager/forms.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
import datetime
from django import forms
from django.core.exceptions import ValidationError
from django.utils.translation import gettext_lazy as _
from kitamanager.sage_payroll import SagePayrolls


class HistoryDateForm(forms.Form):
Expand All @@ -19,3 +22,18 @@ class EmployeeBonusPaymentForm(forms.Form):

pay = forms.DecimalField(max_digits=10, decimal_places=2)
year = forms.IntegerField(initial=datetime.date.today().year)


def _validate_sage_payroll(value):
try:
SagePayrolls(value)
except Exception:
raise ValidationError(_(f"Can not read file {value}. This should be a PDF file. Wrong format?"))


class EmployeeCheckSagePayrollForm(forms.Form):
"""
A Form to upload a .pdf (a Sage Payroll)that will be checked
"""

file_pdf = forms.FileField(validators=[_validate_sage_payroll])
81 changes: 81 additions & 0 deletions django-kitamanager/kitamanager/sage_payroll.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
from dataclasses import dataclass
from dateparser import parse
from pypdf import PdfReader
from decimal import Decimal


@dataclass
class PayrollPerson:
"""
A Person from a Payroll
"""

first_name: str
last_name: str
hours: int
pay_group: int
pay_level: int


class SagePayrolls:
"""
Parse a Sage Payroll (Lohn/Gehaltsabrechnung) PDF file with multiple pages. Each page contains a single person
The file is expected to be in German and currently very eenemeene specific
"""

def __init__(self, lohnscheine_path: str):
self._lohnscheine_path = lohnscheine_path
self._reader = PdfReader(self._lohnscheine_path)
self._date = None

def _get_date(self, text_split):
for count, line in enumerate(text_split):
if line.endswith("Abrechnungsmonat:"):
date_str = text_split[count + 1].split("#")[0]
date = parse(date_str).replace(day=1)
return date
return None

def _get_person(self, text_split):
for count, line in enumerate(text_split):
# FIXME: very eenemeene specific
if line.startswith("TV-EM"):
parts = line.split(" ")
assert parts[0] == "TV-EM"
pay_group = int(parts[1].replace("S", ""))
assert pay_group >= 3 and pay_group <= 10
pay_level = int(parts[2].replace("(", "").replace(")", ""))
assert pay_level >= 1 and pay_level <= 6
hours = Decimal(parts[4])
assert hours >= 0 and hours <= 40
first_name = parts[5][1:]
last_name = " ".join(parts[6:])
return PayrollPerson(
first_name=first_name, last_name=last_name, hours=hours, pay_group=pay_group, pay_level=pay_level
)
return None

@property
def date(self):
# cache the date
if not self._date:
# just use the first page and assume that all pages are for the same date
page = self._reader.pages[0]
text = page.extract_text()
text_split = text.splitlines()
self._date = self._get_date(text_split)
return self._date

@property
def persons(self):
"""
Parse the different PDF pages and return a list of persons
"""
persons = []
for page_number, page in enumerate(self._reader.pages):
text = page.extract_text()
text_split = text.splitlines()
person = self._get_person(text_split)
if person:
persons.append(person)
return persons
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,9 @@
<a class="navbar-item" href="{% url 'kitamanager:employee-statistics' %}">
{% translate "Statistics" %}
</a>
<a class="navbar-item" href="{% url 'kitamanager:employee-check-sage-payroll' %}">
{% translate "Payroll check" %}
</a>
</div>
</div>
<div class="navbar-item has-dropdown is-hoverable">
Expand Down Expand Up @@ -73,6 +76,9 @@
<a class="navbar-item" href="{% url 'kitamanager:bankaccount-list' %}">
{% translate "bank accounts" %}
</a>
<a class="navbar-item" href="{% url 'kitamanager:employee-check-sage-payroll' %}">
{% translate "check Sage payroll" %}
</a>
</div>
</div>
<a class="navbar-item" href="{% url 'admin:index' %}">
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
{% extends "kitamanager/base.html" %}
{% load i18n %}
{% block content %}

<div class="box">
<h5 class="title">
{% translate "check Sage payroll" %}
</h5>
<div class="notification is-info is-light">
{% blocktranslate trimmed %}
The payroll check can be used to compare a Sage payroll (usually sent from Daks)
with the data available in Kitamanager
{% endblocktranslate %}
</div>
<form enctype="multipart/form-data" action="{% url 'kitamanager:employee-check-sage-payroll' %}" method="POST">
{% csrf_token %}
{{ form.as_div }}
<input type="submit" value={% translate "check" %}>
</form>
</div>
{% if data %}
<div class="box">
<h5 class="title">
{% translate "Compare for " %}{{ payroll_date|date:"M Y" }}
</h5>
<div class="table-container">
<table class="table is-striped is-hoverable">
<thead>
<tr>
<th>{% translate "name" %}</th>
<th>{% translate "payroll" %}</th>
<th>{% translate "kitamanager" %}</th>
</tr>
</thead>
<tbody>
{% for person, d in data.items %}
<tr {% if d.payroll != d.kitamanager %}class="has-text-danger"{% endif %}>
<td>{{ person }}</td>
{% if d.payroll %}
<td>{% translate "hours" %}: {{ d.payroll.hours|floatformat:2 }}, {% translate "pay group" %}: {{ d.payroll.pay_group }}, {% translate "pay level" %}: {{ d.payroll.pay_level }}</td>
{% else %}
<td>-</td>
{% endif %}
{% if d.kitamanager %}
<td>{% translate "hours" %}: {{ d.kitamanager.hours|floatformat:2 }}, {% translate "pay group" %}: {{ d.kitamanager.pay_group }}, {% translate "pay level" %}: {{ d.kitamanager.pay_level }}</td>
{% else %}
<td>-</td>
{% endif %}
</tr>
{% endfor %}
<tbody>
</table>
</div>
</div>
{% endif %}

{% endblock %}
7 changes: 7 additions & 0 deletions django-kitamanager/kitamanager/tests/test_employee_urls.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,13 @@ def test_employeepayment_detail(admin_client):
assert response.status_code == 200


@pytest.mark.django_db
def test_employee_check_sage_payroll(admin_client):
# should be 404 for an unknown EmployeePaymentPlan
response = admin_client.get(reverse("kitamanager:employee-check-sage-payroll"))
assert response.status_code == 200


@pytest.mark.django_db
def test_employee_charts_hours_group_by_area(admin_client):
"""
Expand Down
2 changes: 2 additions & 0 deletions django-kitamanager/kitamanager/urls.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
employee_bonuspayment,
employeepayment_list,
employeepayment_detail,
employee_check_sage_payroll,
employee_charts_count_group_by_area,
employee_charts_hours_group_by_area,
)
Expand Down Expand Up @@ -43,6 +44,7 @@
path("employee/", employee_list, name="employee-list"),
path("employee/statistics/", employee_statistics, name="employee-statistics"),
path("employee/bonus/", employee_bonuspayment, name="employee-bonuspayment"),
path("employee/check-sage-payroll", employee_check_sage_payroll, name="employee-check-sage-payroll"),
path("employee/<int:pk>/", employee_detail, name="employee-detail"),
path(
"employee/charts/count-group-by-area/",
Expand Down
52 changes: 51 additions & 1 deletion django-kitamanager/kitamanager/views_employee.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@
from django.shortcuts import render, get_object_or_404
from django import forms
from kitamanager.models import EmployeeContract, Employee, EmployeePaymentPlan, EmployeePaymentTable
from kitamanager.forms import HistoryDateForm, EmployeeBonusPaymentForm
from kitamanager.forms import HistoryDateForm, EmployeeBonusPaymentForm, EmployeeCheckSagePayrollForm
from kitamanager.sage_payroll import SagePayrolls
from django.utils.translation import gettext_lazy as _
import datetime
from django.http import JsonResponse
Expand Down Expand Up @@ -155,6 +156,55 @@ def employee_bonuspayment(request):
)


def _payrolls_data(payrolls):
"""
build datastructure for data from payrolls and kitamanager
"""
data = dict()
# collect data for employees from kitamanager
for employee in EmployeeContract.objects.by_date(payrolls.date):
data[f"{employee.person.last_name}, {employee.person.first_name}"] = dict()
data[f"{employee.person.last_name}, {employee.person.first_name}"]["kitamanager"] = {
"hours": employee.hours_total,
"pay_group": employee.pay_group,
"pay_level": employee.pay_level,
}
# collect data for employees from payrolls
for employee in payrolls.persons:
d = data.get(f"{employee.last_name}, {employee.first_name}", {})
d["payroll"] = {
"hours": employee.hours,
"pay_group": employee.pay_group,
"pay_level": employee.pay_level,
}
data[f"{employee.last_name}, {employee.first_name}"] = d
return data


@login_required
def employee_check_sage_payroll(request):
"""
View to check a Sage Payroll
"""
data = None
payroll_date = None
if request.method == "POST":
form = EmployeeCheckSagePayrollForm(request.POST, request.FILES)
if form.is_valid():
f = form.cleaned_data["file_pdf"]
payrolls = SagePayrolls(f)
data = _payrolls_data(payrolls)
payroll_date = payrolls.date
else:
form = EmployeeCheckSagePayrollForm()

return render(
request,
"kitamanager/employee_check_sage_payroll.html",
{"form": form, "data": data, "payroll_date": payroll_date},
)


@login_required
def employee_charts_count_group_by_area(request):
"""
Expand Down
3 changes: 2 additions & 1 deletion django-kitamanager/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@ dependencies = [
"excel-formulas-calculator",
"django-cors-headers",
"django-debug-toolbar",

"pypdf",
"dateparser",
]
dynamic = ["version"]

Expand Down
2 changes: 1 addition & 1 deletion tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ commands =
deps=
mypy
commands=
python -m pip install types-requests types-PyYAML types-python-dateutil
python -m pip install types-requests types-PyYAML types-python-dateutil types-dateparser
mypy --ignore-missing-imports --allow-subclassing-any --exclude kitamanager/migrations/* .

[testenv:migrationscheck]
Expand Down
Loading