RosterChief/management/bulk_import.py

"""Mass-uploading members from an Excel template: extracting a workbook into plain
row data, and validating that data into what will be created.

Kept as two separate functions so binary parsing happens exactly once (at upload
time) while validation -- the part that must behave identically whether it's
building the preview or actually creating records -- runs against plain data both
times (see MemberImportView / MemberImportConfirmView in management/views.py).
"""

from datetime import date, datetime

import openpyxl
from django.db.models import Q
from django.utils.translation import gettext_lazy as _
from openpyxl.worksheet.datavalidation import DataValidation

from club.models import ClubMembership
from members.models import FamilyMembership, Member

from .forms import MemberForm

TEMPLATE_COLUMNS = ["first_name", "last_name", "date_of_birth", "email", "phone", "emergency_phone", "license", "status", "fee_status", "family_group", "family_role", "membership_kind"]
REQUIRED_HEADER_COLUMNS = {"first_name", "last_name"}

TEMPLATE_EXAMPLE_ROWS = [
    # A standalone member -- no family link.
    ["Alex", "Morgan", date(2012, 5, 14), "alex.morgan@example.com", "+32470123456", "+32470654321", "", ClubMembership.StatusChoices.ACTIVE, ClubMembership.FeeStatus.UNPAID, "", "", ClubMembership.Kind.MEMBER],
    # A parent and child linked together: same family_group value, one row each.
    # The child has no email of its own -- it gets a login only if given one via
    # the "Grant login" action later, same as adding a family by hand.
    #
    # membership_kind sits next to family_role because it answers the question
    # family_role raises: this parent is a `guardian`, so they hold the login and
    # can be contacted but don't count as a member and owe no fee. Put `member`
    # there instead for a parent who also plays -- the two are independent, which
    # is why it is its own column rather than inferred from family_role.
    ["Taylor", "Doe", "", "taylor.doe@example.com", "+32470654322", "", "", ClubMembership.StatusChoices.ACTIVE, "", "Doe family", FamilyMembership.FamilyRole.PARENT, ClubMembership.Kind.GUARDIAN],
    ["Jamie", "Doe", date(2014, 3, 2), "", "", "+32470654322", "", ClubMembership.StatusChoices.ACTIVE, ClubMembership.FeeStatus.UNPAID, "Doe family", FamilyMembership.FamilyRole.CHILD, ClubMembership.Kind.MEMBER],
]


def build_member_import_template():
    """The downloadable .xlsx: header row, one example row, and a dropdown on the
    status/fee_status columns so a cell can't be typo'd into an invalid value."""
    workbook = openpyxl.Workbook()
    sheet = workbook.active
    sheet.title = "Members"

    sheet.append(TEMPLATE_COLUMNS)
    for cell in sheet[1]:
        cell.font = openpyxl.styles.Font(bold=True)
    sheet.freeze_panes = "A2"

    for example_row in TEMPLATE_EXAMPLE_ROWS:
        sheet.append(example_row)

    for column_name, choices in (("membership_kind", ClubMembership.Kind), ("status", ClubMembership.StatusChoices), ("fee_status", ClubMembership.FeeStatus), ("family_role", FamilyMembership.FamilyRole)):
        column_index = TEMPLATE_COLUMNS.index(column_name) + 1
        column_letter = sheet.cell(row=1, column=column_index).column_letter
        options = ",".join(choices.values)
        validation = DataValidation(type="list", formula1=f'"{options}"', allow_blank=True)
        sheet.add_data_validation(validation)
        validation.add(f"{column_letter}2:{column_letter}1000")

    for column_index, column_name in enumerate(TEMPLATE_COLUMNS, start=1):
        sheet.column_dimensions[sheet.cell(row=1, column=column_index).column_letter].width = max(12, len(column_name) + 2)

    return workbook


def read_member_import_workbook(file):
    """Uploaded .xlsx -> list[dict], one dict per row keyed by TEMPLATE_COLUMNS.

    Normalizes every cell to a plain str/None here: openpyxl returns typed cells
    (a date-formatted cell comes back as a datetime.date, a phone number typed as
    digits-only can come back as a number), so this is the one place that has to
    deal with that -- everything downstream, including the session, only ever
    sees plain strings.
    """
    workbook = openpyxl.load_workbook(file, data_only=True)
    sheet = workbook.active

    header = [str(cell.value).strip() if cell.value is not None else "" for cell in next(sheet.iter_rows(min_row=1, max_row=1))]
    if not REQUIRED_HEADER_COLUMNS.issubset(header):
        missing = REQUIRED_HEADER_COLUMNS - set(header)
        raise ValueError(_("This doesn't look like the template — missing column(s): %(columns)s.") % {"columns": ", ".join(sorted(missing))})

    rows = []
    for excel_row in sheet.iter_rows(min_row=2):
        values = {}
        for column_name, cell in zip(header, excel_row, strict=False):
            if column_name not in TEMPLATE_COLUMNS:
                continue
            values[column_name] = _cell_to_str(cell.value)

        if not any(values.values()):
            continue  # a fully blank row (trailing spreadsheet padding) isn't a row to import
        rows.append(values)

    return rows


def _cell_to_str(value):
    if value is None:
        return ""
    if isinstance(value, datetime):
        return value.date().isoformat()
    if isinstance(value, date):
        return value.isoformat()
    if isinstance(value, float) and value.is_integer():
        return str(int(value))
    return str(value).strip()


def parse_member_import_rows(rows, club):
    """list[dict] (as returned by read_member_import_workbook) -> one result per
    row: {line_number, raw, member, membership_kwargs, errors}. `member` is an
    unsaved Member instance (ready to .save()) when the row is valid, else None."""
    results = []
    seen_emails = set()

    for line_number, raw in enumerate(rows, start=2):
        errors = []
        member_fields = {key: raw.get(key, "") for key in ("first_name", "last_name", "date_of_birth", "email", "phone", "emergency_phone")}
        form = MemberForm(data=member_fields)

        member = None
        if form.is_valid():
            member = form.save(commit=False)
        else:
            for field_errors in form.errors.values():
                errors.extend(field_errors)

        email = member_fields["email"].strip()
        if email:
            if email.lower() in seen_emails:
                errors.append(_("Duplicate email in this file."))
            seen_emails.add(email.lower())
            already_in_club = Member.objects.filter(member_of__club=club).filter(Q(email__iexact=email) | Q(user__email__iexact=email)).exists()
            if already_in_club:
                errors.append(_("Already a member of this club."))

        membership_kwargs, status_fee_errors = _parse_membership_fields(raw)
        errors.extend(status_fee_errors)

        family_group, family_role, family_errors = _parse_family_fields(raw)
        errors.extend(family_errors)

        # The two columns are otherwise independent -- a parent may or may not also
        # be a member -- but a child is the member the guardian is attached *to*,
        # so that particular combination is always a mistake.
        if family_role == FamilyMembership.FamilyRole.CHILD and membership_kwargs["kind"] == ClubMembership.Kind.GUARDIAN:
            errors.append(_("A child is always a member, so membership_kind cannot be 'guardian'."))

        results.append(
            {
                "line_number": line_number,
                "raw": raw,
                "member": member if not errors else None,
                "membership_kwargs": membership_kwargs,
                "family_group": family_group,
                "family_role": family_role,
                "errors": errors,
            }
        )

    return results


def _parse_membership_fields(raw):
    errors = []
    license_number = raw.get("license", "").strip()

    kind = raw.get("membership_kind", "").strip()
    if kind:
        kind_value = _match_choice(kind, ClubMembership.Kind)
        if kind_value is None:
            errors.append(_("Invalid membership_kind '%(value)s'.") % {"value": kind})
    else:
        # Blank means member: the overwhelmingly common row, and the value every
        # file written before this column existed effectively carried.
        kind_value = ClubMembership.Kind.MEMBER

    status = raw.get("status", "").strip()
    if status:
        status_value = _match_choice(status, ClubMembership.StatusChoices)
        if status_value is None:
            errors.append(_("Invalid status '%(value)s'.") % {"value": status})
    else:
        status_value = ClubMembership.StatusChoices.ACTIVE

    fee_status = raw.get("fee_status", "").strip()
    if fee_status:
        fee_status_value = _match_choice(fee_status, ClubMembership.FeeStatus)
        if fee_status_value is None:
            errors.append(_("Invalid fee status '%(value)s'.") % {"value": fee_status})
    else:
        fee_status_value = ClubMembership.FeeStatus.UNPAID

    return {"kind": kind_value, "license": license_number, "status": status_value, "fee_status": fee_status_value}, errors


def _parse_family_fields(raw):
    """family_group is a freeform label -- rows sharing the same non-blank value
    (within this file only; it isn't matched against families already in the
    club) are linked into one family. family_role decides whether the row gets a
    login: PARENT/GUARDIAN do (via their email), CHILD/OTHER don't -- same as
    registering a family by hand (see members.services.family)."""
    errors = []
    family_group = raw.get("family_group", "").strip()
    family_role_raw = raw.get("family_role", "").strip()

    if not family_group:
        # family_role=child on its own is the migration case this exists for: a
        # child the club holds with no parent on file yet. It gets a family of
        # its own so there is something for a parent to join later -- see
        # members.services.claims. Any other lone role is still a mistake.
        if family_role_raw:
            family_role = _match_choice(family_role_raw, FamilyMembership.FamilyRole)
            if family_role == FamilyMembership.FamilyRole.CHILD:
                return "", family_role, errors
            errors.append(_("family_role given without a family_group. Only 'child' is allowed on its own, for a child whose parent will register later."))
        return "", None, errors

    if not family_role_raw:
        errors.append(_("family_role is required when family_group is set."))
        return family_group, None, errors

    family_role = _match_choice(family_role_raw, FamilyMembership.FamilyRole)
    if family_role is None:
        errors.append(_("Invalid family_role '%(value)s'.") % {"value": family_role_raw})

    return family_group, family_role, errors


def _match_choice(value, choices):
    for choice_value in choices.values:
        if choice_value.lower() == value.lower():
            return choice_value
    return None