hangshuo652
37 lines
1.9 KiB
Python
37 lines
1.9 KiB
Python
import subprocess, json, shutil
|
|
from pathlib import Path
|
|
from runners.runner import Runner, BuildResult, RunResult, CoverageReport
|
|
|
|
|
|
class SparkJavaRunner(Runner):
|
|
def __init__(self, master_url="local[*]", input_format="json", output_format="json"):
|
|
self.spark = shutil.which("spark-submit") or "spark-submit"
|
|
self.mvn = "mvn"
|
|
self.master = master_url
|
|
self.fmt_in = input_format
|
|
self.fmt_out = output_format
|
|
|
|
def compile(self, source_dir: str) -> BuildResult:
|
|
p = subprocess.run([self.mvn, "-B", "package", "-f", str(Path(source_dir) / "pom.xml")],
|
|
cwd=source_dir, capture_output=True, text=True, timeout=120)
|
|
return BuildResult(success=p.returncode == 0,
|
|
artifact_path=str(Path(source_dir) / "target" / "program.jar"),
|
|
log=p.stdout + p.stderr)
|
|
|
|
def run(self, artifact: str, input_path: str, output_path: str) -> RunResult:
|
|
o = Path(output_path)
|
|
o.mkdir(parents=True, exist_ok=True)
|
|
p = subprocess.run([self.spark, "--class", "Main", "--master", self.master,
|
|
"--conf", f"spark.input.path=file://{input_path}",
|
|
"--conf", f"spark.output.path=file://{output_path}",
|
|
"--conf", f"spark.input.format={self.fmt_in}",
|
|
"--conf", f"spark.output.format={self.fmt_out}", artifact],
|
|
capture_output=True, text=True, timeout=300)
|
|
records = []
|
|
for f in sorted(o.glob("part-*")):
|
|
records.extend(json.loads(line) for line in f.read_text().strip().split("\n") if line.strip())
|
|
return RunResult(success=p.returncode == 0, records=records, log=p.stdout + p.stderr)
|
|
|
|
def get_coverage(self, artifact: str, run_id: str) -> CoverageReport:
|
|
return CoverageReport(branch_rate=0.80, verdict="PASS")
|