From d85c25b21e4996be8f278ef92e789be3887b190d Mon Sep 17 00:00:00 2001 From: Jordan Hewitt Date: Wed, 20 Mar 2024 09:42:18 -0700 Subject: [PATCH] began writing my own matching alg. --- lib/libdart_openmvg_generated.dart | 22 ++++++ scripts/compilelib.sh | 4 +- src/extlib/openMVG | 2 +- src/frame.cxx | 112 ++++++++++++++++++++++------- src/frame.h | 73 ++++++++++++++++++- src/image.cxx | 58 ++------------- src/image.h | 2 +- src/streamingview.cxx | 3 +- 8 files changed, 190 insertions(+), 86 deletions(-) diff --git a/lib/libdart_openmvg_generated.dart b/lib/libdart_openmvg_generated.dart index dd6743e..2dce9e5 100644 --- a/lib/libdart_openmvg_generated.dart +++ b/lib/libdart_openmvg_generated.dart @@ -42,6 +42,28 @@ class LibDartOpenMVG { late final _make_buffer = _make_bufferPtr .asFunction Function(ffi.Pointer, int)>(); + int read_buffer( + ffi.Pointer arg0, + ffi.Pointer> arg1, + ffi.Pointer arg2, + ) { + return _read_buffer( + arg0, + arg1, + arg2, + ); + } + + late final _read_bufferPtr = _lookup< + ffi.NativeFunction< + ffi.Int Function( + ffi.Pointer, + ffi.Pointer>, + ffi.Pointer)>>('read_buffer'); + late final _read_buffer = _read_bufferPtr.asFunction< + int Function(ffi.Pointer, ffi.Pointer>, + ffi.Pointer)>(); + ffi.Pointer new_frame_from_handle( ffi.Pointer arg0, int arg1, diff --git a/scripts/compilelib.sh b/scripts/compilelib.sh index 9cd03c4..fb65a54 100755 --- a/scripts/compilelib.sh +++ b/scripts/compilelib.sh @@ -24,6 +24,4 @@ cmake \ -DCMAKE_BUILD_TYPE=Debug make -printf "\033c" - -set +e \ No newline at end of file +set +e diff --git a/src/extlib/openMVG b/src/extlib/openMVG index aff37dd..7fd19fc 160000 --- a/src/extlib/openMVG +++ b/src/extlib/openMVG @@ -1 +1 @@ -Subproject commit aff37dd005f85b3589aa50caca13b74ca7bd57e4 +Subproject commit 7fd19fc22419a35481d39d2946f77e1dc00df37c diff --git a/src/frame.cxx b/src/frame.cxx index 0bb2cf3..f9e7b1e 100644 --- a/src/frame.cxx +++ b/src/frame.cxx @@ -1,5 +1,5 @@ #include "frame.h" - +#include FILE *make_buffer(const uint8_t *data, const size_t data_len) { @@ -20,43 +20,46 @@ FILE *make_buffer(const uint8_t *data, const size_t data_len) return file; } -int read_buffer(FILE *file, uint8_t **buffer, size_t *length) { - if (file != NULL) { - // Get the file size - fseek(file, 0, SEEK_END); - (*length) = ftell(file); - fseek(file, 0, SEEK_SET); +int read_buffer(FILE *file, uint8_t **buffer, size_t *length) +{ + if (file != NULL) + { + // Get the file size + fseek(file, 0, SEEK_END); + (*length) = ftell(file); + fseek(file, 0, SEEK_SET); - // Allocate memory - *buffer = (uint8_t *) malloc((*length) * sizeof(char)); - if (buffer == NULL) { - printf("Error: Unable to allocate memory.\n"); - return 1; - } + // Allocate memory + *buffer = (uint8_t *)malloc((*length) * sizeof(char)); + if (buffer == NULL) + { + printf("Error: Unable to allocate memory.\n"); + return 1; + } - // Read file - size_t newLen = fread(*buffer, sizeof(char), *length, file); - if (newLen == 0) { - if (feof(file)) { - printf("Error: End of file reached.\n"); - return 2; - } - printf("Error: Reading file.\n"); - return 3; + // Read file + size_t newLen = fread(*buffer, sizeof(char), *length, file); + if (newLen == 0) + { + if (feof(file)) + { + printf("Error: End of file reached.\n"); + return 2; } - return 4; + printf("Error: Reading file.\n"); + return 3; + } + return 4; } return 5; } - Frame *new_frame_from_data(const uint8_t *data, const size_t data_len, int w, int h, int depth) { FILE *buf = make_buffer(data, data_len); return new_frame_from_handle(buf, w, h, depth); } - Frame *new_frame_from_handle(FILE *stream, int w, int h, int depth) { Frame *f = (Frame *)malloc(sizeof(Frame)); @@ -65,4 +68,61 @@ Frame *new_frame_from_handle(FILE *stream, int w, int h, int depth) f->h = h; f->depth = depth; return f; -} \ No newline at end of file +} + +#ifdef __cplusplus + +openMVG::image::Image DartOpenMvg::imageFromFrame(const CFrame *frame) +{ + rewind(frame->stream); + std::vector imageData; + if (!openMVG::image::ReadPngStream(frame->stream, &imageData, (int *)&(frame->w), (int *)&(frame->h), (int *)&(frame->depth))) + { + throw std::runtime_error("Could not read stream"); + }; + int w = frame->w, h = frame->h; + Eigen::Matrix eigenMatrix(frame->w, frame->h); + for (int i = 0; i < w * h; ++i) + { + eigenMatrix(i / h, i % h) = imageData[i]; + } + + return openMVG::image::Image(eigenMatrix); +} + +void DartOpenMvg::Frame::calculateFeatures() +{ + using namespace openMVG::image; + using namespace openMVG::features; + auto desc = SIFT_Anatomy_Image_describer(SIFT_Anatomy_Image_describer::Params()); + auto image = imageFromFrame(cFrame); + mRegions = desc.Describe(image); +} + +openMVG::image::Image DartOpenMvg::Frame::getMvgImage() { + return imageFromFrame(this->cFrame); +} + +void DartOpenMvg::Frames::computeMatches() +{ + using namespace openMVG::matching; + using namespace openMVG::matching_image_collection; + using namespace openMVG::sfm; + using namespace openMVG::image; + Matcher *matcher = nullptr; + + Regions_Provider provider = Regions_Provider(); + + matcher = new Matcher_Regions(0.8f, CASCADE_HASHING_L2); + + openMVG::Pair_Set pairs = openMVG::Pair_Set(); + Image im1 = this->at(0).getMvgImage(); + Image im2 = this->at(1).getMvgImage(); + auto pair = openMVG::Pair(0, 1); + pairs.insert(pair); + + PairWiseMatches map_punitive_matches; + + // matcher->Match(provider, pairs, map_punitive_matches, NULL); + +#endif // __cplusplus \ No newline at end of file diff --git a/src/frame.h b/src/frame.h index 1481360..0b6b46c 100644 --- a/src/frame.h +++ b/src/frame.h @@ -1,12 +1,30 @@ #include #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include "./base.hpp" #ifndef __FRAME_H #define __FRAME_H - typedef struct _Frame { FILE *stream; @@ -16,9 +34,60 @@ typedef struct _Frame } Frame; FILE *make_buffer(const uint8_t *, const size_t); - int read_buffer(FILE *, uint8_t **, size_t *); +#ifdef __cplusplus + +typedef Frame CFrame; + +namespace DartOpenMvg +{ + openMVG::image::Image imageFromFrame(const Frame *); + + class InMemoryMatcherRegions : public openMVG::matching_image_collection::Matcher { + void Match( + const Pair_Set &, + PariWiseMatchesContainer &, + ) const override; + }; + + class Frame + { + protected: + const CFrame *cFrame; + std::stringstream mStream; + int mW; + int mH; + int mDepth; + std::unique_ptr mRegions; + + public: + Frame(const _Frame *f) : mW(f->w), mH(f->h), mDepth(f->depth) + { + cFrame = f; + uint8_t *buffer_data = NULL; + size_t len = 0; + read_buffer(f->stream, &buffer_data, &len); + mStream = std::stringstream(std::string((char *)buffer_data)); + } + openMVG::image::Image DartOpenMvg::Frame::getMvgImage(); + void calculateFeatures(); + }; + + typedef std::pair FrameMatchKey; + typedef std::map FrameFeatureMatch; + + class Frames : std::vector + { + + protected: + FrameFeatureMatch mFeatureMap; + public: + void computeMatches(); + }; +} +#endif + _FFI_PLUGIN Frame *new_frame_from_handle(FILE *, int, int, int); diff --git a/src/image.cxx b/src/image.cxx index 04e3b25..cce56cd 100644 --- a/src/image.cxx +++ b/src/image.cxx @@ -38,7 +38,7 @@ void free_image_result(ImageResult *image_result) } -std::vector Archimedes::get_image_data_as_vector(const Frame *frame) +std::vector DartOpenmvg::get_image_data_as_vector(const Frame *frame) { rewind(frame->stream); std::vector imageData; @@ -51,7 +51,7 @@ std::vector Archimedes::get_image_data_as_vector(const Frame *frame) #ifdef __cplusplus -ImageResult *Archimedes::get_image_data(const Frame *frame) +ImageResult *DartOpenmvg::get_image_data(const Frame *frame) { try { const std::vector imageData = get_image_data_as_vector(frame); @@ -69,58 +69,12 @@ ImageResult *Archimedes::get_image_data(const Frame *frame) _FFI_PLUGIN ImageResult *archimedes_get_image_data(const Frame *frame) { - return Archimedes::get_image_data(frame); + return DartOpenmvg::get_image_data(frame); } -int Archimedes::images_to_sfm(const Frame **frames, size_t n_frames) +int DartOpenmvg::images_to_sfm(const Frame **frames, size_t n_frames) { - // https://openmvg.readthedocs.io/en/latest/openMVG/sfm/sfm/#structure-computation-from-known-camera-poses - using namespace openMVG::sfm; - - const Frame *frame = frames[0]; - const std::vector imageData = get_image_data_as_vector(frame); - - const size_t w = frame->w; - const size_t h = frame->h; - - // Create the Eigen matrix and the image - - Eigen::Matrix eigenMatrix(w, h); - for (int i = 0; i < w * h; ++i) { - eigenMatrix(i / h, i % h) = imageData[i]; - } - - openMVG::image::Image bvgImage(eigenMatrix); - - // Extract the features from the image - openMVG::features::SIFT_Anatomy_Image_describer desc; - std::unique_ptr siftanatomy = desc.Describe_SIFT_Anatomy(bvgImage); - openMVG::features::SIOPointFeatures features = siftanatomy.get()->Features(); - - openMVG::features::SIOPointFeature feature = features.at(0); - - Eigen::MatrixBase mat1 = feature.coords().matrix(); - - openMVG::TriangulateNView(mat1); - - // SfM_Data sfmData {}; - // sfmData.views = Views(); - - openMVG::sfm::GlobalSfM_Rotation_AveragingSolver rotSolver; - openMVG::sfm::GlobalSfM_Translation_AveragingSolver tranSolver; - - openMVG::sfm::SfM_Data_Structure_Computation_Blind computation; - - - openMVG::rotation_averaging::RelativeRotation relRot; - openMVG::rotation_averaging::RelativeRotations relativeRot_in; - - // rotSolver.Run( - // openMVG::sfm::ERotationAveragingMethod::ROTATION_AVERAGING_L1, - // openMVG::sfm::ERelativeRotationInferenceMethod::TRIPLET_ROTATION_INFERENCE_COMPOSITION_ERROR, - // relativeRot_in, - // map_globalR, - // ); + // TODO return 0; @@ -129,5 +83,5 @@ int Archimedes::images_to_sfm(const Frame **frames, size_t n_frames) _FFI_PLUGIN int archimedes_images_to_sfm(const Frame **frames, size_t n_frames) { - return Archimedes::images_to_sfm(frames, n_frames); + return DartOpenmvg::images_to_sfm(frames, n_frames); } \ No newline at end of file diff --git a/src/image.h b/src/image.h index 2e903b9..02bba22 100644 --- a/src/image.h +++ b/src/image.h @@ -83,7 +83,7 @@ int image_result_free(ImageResult *); #ifdef __cplusplus -namespace Archimedes +namespace DartOpenmvg { std::vector get_image_data_as_vector(const Frame *frame); ImageResult *get_image_data(const Frame *); diff --git a/src/streamingview.cxx b/src/streamingview.cxx index 101f719..5b83626 100644 --- a/src/streamingview.cxx +++ b/src/streamingview.cxx @@ -4,7 +4,8 @@ void libdart_openmvg::StreamingView::load(cereal::BinaryInputArchive &ar) const { uint8_t *data = NULL; const size_t dLen1 = (mFrame->w * mFrame->h * mFrame->depth * sizeof(uint8_t)); - ar.loadBinary(data, sizeof(uint8_t) * dLen1); + size_t sz1 = sizeof(uint8_t) * dLen1; + ar.loadBinary(data, sz1); const size_t dlen2 = sizeof(data) * sizeof(uint8_t); mFrame->stream = make_buffer(data, dlen2); rewind(mFrame->stream);