20 lines
547 B
Python
20 lines
547 B
Python
"""Module containing necessary functionality to write to the PostgreSQL sink"""
|
|
|
|
import logging
|
|
|
|
import apache_beam as beam
|
|
|
|
from helpers.parsers import parse_row
|
|
|
|
|
|
class ProcessRows(beam.DoFn):
|
|
"""DoFn to process and parse rows from the input file into structured
|
|
dictionaries"""
|
|
|
|
# pylint: disable=abstract-method,arguments-differ
|
|
def process(self, element):
|
|
if (row := parse_row(element)) is not None:
|
|
yield row
|
|
else:
|
|
logging.warning("could not successfully parse this row: %s", element)
|