Hugintrunk  0.1
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
PhotometricOptimizer.cpp
Go to the documentation of this file.
1 // -*- c-basic-offset: 4 -*-
24 #include "PhotometricOptimizer.h"
25 
26 #include <fstream>
27 #include <foreign/levmar/levmar.h>
30 
31 #ifdef DEBUG
32 #define DEBUG_LOG_VIG 1
33 #endif
34 
35 
36 namespace HuginBase {
37 
39 inline double weightHuber(double x, double sigma)
40 {
41  if (x > sigma) {
42  x = sqrt(sigma* (2*x - sigma));
43  }
44  return x;
45 }
46 
47 
48 
50  const std::vector<vigra_ext::PointPairRGB> & data,
51  double mEstimatorSigma, bool symmetric,
52  int maxIter, AppBase::ProgressDisplay* progress)
53  : m_pano(pano), m_data(data), huberSigma(mEstimatorSigma), symmetricError(symmetric),
54  m_maxIter(maxIter), m_progress(progress)
55 {
56  assert(pano.getNrOfImages() == optvars.size());
57 
58  for (unsigned i=0; i < pano.getNrOfImages(); i++) {
59  m_imgs.push_back(pano.getSrcImage(i));
60  }
61 
62  std::vector<std::set<std::string> > usedVars(pano.getNrOfImages());
63 
64  // create variable map with param <-> var assignments
65  for (unsigned i=0; i < optvars.size(); i++)
66  {
67  const std::set<std::string> vars = optvars[i];
68  const SrcPanoImage & img_i = pano.getImage(i);
69  for (std::set<std::string>::const_iterator it = vars.begin();
70  it != vars.end(); ++it)
71  {
72  VarMapping var;
73  var.type = *it;
74  //check if variable is yet included
75  if(set_contains(usedVars[i],var.type))
76  continue;
77  var.imgs.insert(i);
78  usedVars[i].insert(var.type);
79  //now check all linked images and add image nr
80 #define CheckLinked(name)\
81  if(img_i.name##isLinked())\
82  {\
83  for(unsigned j=i+1;j<pano.getNrOfImages();j++)\
84  if(img_i.name##isLinkedWith(pano.getImage(j)))\
85  {\
86  var.imgs.insert(j);\
87  usedVars[j].insert(var.type);\
88  };\
89  }
90 
91  if(var.type=="Eev")
92  {
94  };
95  if(var.type=="Er")
96  {
98  };
99  if(var.type=="Eb")
100  {
102  };
103  if(var.type[0]=='R')
104  {
106  };
107  if(var.type=="Va" || var.type=="Vb" || var.type=="Vc" || var.type=="Vd")
108  {
110  }
111  if(var.type=="Vx" || var.type=="Vy")
112  {
114  };
115 #undef CheckLinked
116  m_vars.push_back(var);
117  }
118  }
119 }
120 
122 {
123  for (size_t i=0; i < m_vars.size(); i++)
124  {
125  assert(!m_vars[i].imgs.empty());
126  // get corresponding image number
127  unsigned j = *(m_vars[i].imgs.begin());
128  // get value
129  x[i] = m_imgs[j].getVar(m_vars[i].type);
130  // TODO: transform some variables, such as the vignetting center!
131  }
132 }
133 
134 
136 {
137  for (size_t i=0; i < m_vars.size(); i++)
138  {
139  // TODO: transform some variables, such as the vignetting center!
140  assert(!m_vars[i].imgs.empty());
141  // copy value int all images
142  for (std::set<unsigned>::const_iterator it = m_vars[i].imgs.begin();
143  it != m_vars[i].imgs.end(); ++it)
144  {
145  m_imgs[*it].setVar(m_vars[i].type, x[i]);
146  }
147  }
148 }
149 
150 
151 
152 void PhotometricOptimizer::photometricError(double *p, double *x, int m, int n, void * data)
153 {
154 #ifdef DEBUG_LOG_VIG
155  static int iter = 0;
156 #endif
158  typedef Photometric::InvResponseTransform<vigra::RGBValue<double>, vigra::RGBValue<double> > InvRespFunc;
159 
160  int xi = 0 ;
161 
162  OptimData * dat = static_cast<OptimData*>(data);
163  dat->FromX(p);
164 #ifdef DEBUG_LOG_VIG
165  std::ostringstream oss;
166  oss << "vig_log_" << iter;
167  iter++;
168  std::ofstream log(oss.str().c_str());
169  log << "VIGparams = [";
170  for (int i = 0; i < m; i++) {
171  log << p[i] << " ";
172  }
173  log << " ]; " << std::endl;
174  // TODO: print parameters of images.
175  std::ofstream script("vig_test.pto");
176  OptimizeVector optvars(dat->m_pano.getNrOfImages());
177  UIntSet imgs = dat->m_pano.getActiveImages();
178  dat->m_pano.printPanoramaScript(script, optvars, dat->m_pano.getOptions(), imgs, false, "");
179 #endif
180 
181  size_t nImg = dat->m_imgs.size();
182  std::vector<RespFunc> resp(nImg);
183  std::vector<InvRespFunc> invResp(nImg);
184  for (size_t i=0; i < nImg; i++) {
185  resp[i] = RespFunc(dat->m_imgs[i]);
186  invResp[i] = InvRespFunc(dat->m_imgs[i]);
187  // calculate the monotonicity error
188  double monErr = 0;
189  if (dat->m_imgs[i].getResponseType() == SrcPanoImage::RESPONSE_EMOR) {
190  // calculate monotonicity error
191  int lutsize = resp[i].m_lutR.size();
192  for (int j=0; j < lutsize-1; j++)
193  {
194  double d = resp[i].m_lutR[j] - resp[i].m_lutR[j+1];
195  if (d > 0) {
196  monErr += d*d*lutsize;
197  }
198  }
199  }
200  x[xi++] = monErr;
201  // enforce a montonous response curves
202  resp[i].enforceMonotonicity();
203  invResp[i].enforceMonotonicity();
204  }
205 
206 #ifdef DEBUG
207  double sqerror=0;
208 #endif
209  // loop over all points to calculate the error
210 #ifdef DEBUG_LOG_VIG
211  log << "VIGval = [ ";
212 #endif
213 
214  for (std::vector<vigra_ext::PointPairRGB>::const_iterator it = dat->m_data.begin();
215  it != dat->m_data.end(); ++it)
216  {
217  vigra::RGBValue<double> l2 = invResp[it->imgNr2](it->i2, it->p2);
218  vigra::RGBValue<double> i2ini1 = resp[it->imgNr1](l2, it->p1);
219  vigra::RGBValue<double> error = it->i1 - i2ini1;
220 
221 
222  // if requested, calcuate the error in image 2 as well.
223  //TODO: weighting dependent on the pixel value? check if outside of i2 range?
224  vigra::RGBValue<double> l1 = invResp[it->imgNr1](it->i1, it->p1);
225  vigra::RGBValue<double> i1ini2 = resp[it->imgNr2](l1, it->p2);
226  vigra::RGBValue<double> error2 = it->i2 - i1ini2;
227 
228 #ifdef DEBUG
229  for (int i=0; i < 3; i++) {
230  sqerror += error[i]*error[i];
231  sqerror += error2[i]*error2[i];
232  }
233 #endif
234 
235  // use huber robust estimator
236  if (dat->huberSigma > 0) {
237  for (int i=0; i < 3; i++) {
238  x[xi++] = weightHuber(fabs(error[i]), dat->huberSigma);
239  x[xi++] = weightHuber(fabs(error2[i]), dat->huberSigma);
240  }
241  } else {
242  x[xi++] = error[0];
243  x[xi++] = error[1];
244  x[xi++] = error[2];
245  x[xi++] = error2[0];
246  x[xi++] = error2[1];
247  x[xi++] = error2[2];
248  }
249 
250 #ifdef DEBUG_LOG_VIG
251  log << it->i1.green() << " "<< l1.green() << " " << i1ini2.green() << " "
252  << it->i2.green() << " "<< l2.green() << " " << i2ini1.green() << "; " << std::endl;
253 #endif
254 
255  }
256 #ifdef DEBUG_LOG_VIG
257  log << std::endl << "VIGerr = [";
258  for (int i = 0; i < n; i++) {
259  log << x[i] << std::endl;
260  }
261  log << " ]; " << std::endl;
262 #endif
263 #ifdef DEBUG
264  DEBUG_DEBUG("squared error: " << sqerror);
265 #endif
266 }
267 
268 int PhotometricOptimizer::photometricVis(double *p, double *x, int m, int n, int iter, double sqerror, void * data)
269 {
270  OptimData * dat = static_cast<OptimData*>(data);
271  char tmp[200];
272  tmp[199] = 0;
273  double error = sqrt(sqerror/n)*255;
274  snprintf(tmp,199, "Iteration: %d, error: %f", iter, error);
275  return dat->m_progress->updateDisplay(std::string(tmp)) ? 1 : 0 ;
276 }
277 
279  const std::vector<vigra_ext::PointPairRGB> & correspondences,
280  const float imageStepSize,
281  AppBase::ProgressDisplay* progress,
282  double & error)
283 {
284 
285  OptimizeVector photometricVars;
286  // keep only the photometric variables
287  unsigned int optCount=0;
288  for (OptimizeVector::const_iterator it=vars.begin(); it != vars.end(); ++it)
289  {
290  std::set<std::string> cvars;
291  for (std::set<std::string>::const_iterator itv = (*it).begin();
292  itv != (*it).end(); ++itv)
293  {
294  if ((*itv)[0] == 'E' || (*itv)[0] == 'R' || (*itv)[0] == 'V') {
295  cvars.insert(*itv);
296  }
297  }
298  photometricVars.push_back(cvars);
299  optCount+=cvars.size();
300  }
301  //if no variables to optimize return
302  if(optCount==0)
303  {
304  return;
305  };
306 
307  int nMaxIter = 250;
308  OptimData data(pano, photometricVars, correspondences, 5 * imageStepSize, false, nMaxIter, progress);
309 
310  double info[LM_INFO_SZ];
311 
312  // parameters
313  int m=data.m_vars.size();
314  vigra::ArrayVector<double> p(m, 0.0);
315 
316  // vector for errors
317  int n=2*3*correspondences.size()+pano.getNrOfImages();
318  vigra::ArrayVector<double> x(n, 0.0);
319 
320  data.ToX(p.begin());
321 #ifdef DEBUG
322  printf("Parameters before optimisation: ");
323  for(int i=0; i<m; ++i)
324  printf("%.7g ", p[i]);
325  printf("\n");
326 #endif
327 
328  // TODO: setup optimisation options with some good defaults.
329  double optimOpts[5];
330 
331  optimOpts[0] = LM_INIT_MU; // init mu
332  // stop thresholds
333  optimOpts[1] = LM_STOP_THRESH; // ||J^T e||_inf
334  optimOpts[2] = LM_STOP_THRESH; // ||Dp||_2
335  optimOpts[3] = std::pow(imageStepSize*0.1f, 2); // ||e||_2
336  // difference mode
337  optimOpts[4] = LM_DIFF_DELTA;
338 
339  dlevmar_dif(&photometricError, &photometricVis, &(p[0]), &(x[0]), m, n, nMaxIter, optimOpts, info, NULL,NULL, &data); // no jacobian
340 
341  // copy to source images (data.m_imgs)
342  data.FromX(p.begin());
343  // calculate error at solution
344  data.huberSigma = 0;
345  photometricError(&(p[0]), &(x[0]), m, n, &data);
346  error = 0;
347  for (int i=0; i<n; i++) {
348  error += x[i]*x[i];
349  }
350  error = sqrt(error/n);
351 
352 #ifdef DEBUG
353  printf("Levenberg-Marquardt returned in %g iter, reason %g\nSolution: ", info[5], info[6]);
354  for(int i=0; i<m; ++i)
355  printf("%.7g ", p[i]);
356  printf("\n\nMinimization info:\n");
357  for(int i=0; i<LM_INFO_SZ; ++i)
358  printf("%g ", info[i]);
359  printf("\n");
360 #endif
361 
362  // copy settings to panorama
363  for (unsigned i=0; i<pano.getNrOfImages(); i++) {
364  pano.setSrcImage(i, data.m_imgs[i]);
365  }
366 }
367 
368 bool IsHighVignetting(std::vector<double> vigCorr)
369 {
371  srcImage.setRadialVigCorrCoeff(vigCorr);
372  srcImage.setSize(vigra::Size2D(500, 500));
373  Photometric::ResponseTransform<double> transform(srcImage);
374  for (size_t x = 0; x < 250; x += 10)
375  {
376  const double vigFactor = transform.calcVigFactor(hugin_utils::FDiff2D(x, x));
377  if (vigFactor < 0.2 || vigFactor > 1.1)
378  {
379  return true;
380  };
381  };
382  return false;
383 };
384 
386 {
387  for(size_t i=0; i<pano.getNrOfImages(); i++)
388  {
389  if(pano.getImage(i).getWhiteBalanceBlue()>3)
390  {
391  return true;
392  };
393  if(pano.getImage(i).getWhiteBalanceRed()>3)
394  {
395  return true;
396  };
397  };
398  return false;
399 };
400 
402  const std::vector<vigra_ext::PointPairRGB> & correspondences,
403  const float imageStepSize,
404  AppBase::ProgressDisplay* progress,
405  double & error)
406 {
407  PanoramaOptions opts = pano.getOptions();
408  UIntSet images;
409  fill_set(images, 0, pano.getNrOfImages()-1);
410  std::vector<UIntSet> stacks = getHDRStacks(pano, images, pano.getOptions());
411  const bool singleStack = (stacks.size() == 1);
412 
413  int vars = 0;
414  if (mode == OPT_PHOTOMETRIC_LDR || mode == OPT_PHOTOMETRIC_LDR_WB)
415  {
416  // optimize exposure
417  vars = OPT_EXP;
418  optimizePhotometric(pano,
419  createOptVars(pano, vars, opts.colorReferenceImage),
420  correspondences, imageStepSize, progress, error);
421  };
422 
423  if(!singleStack)
424  {
425  //optimize vignetting only if there are more than 1 image stack
426  // for a single stack vignetting can't be calculated by this optimization
427  vars |= OPT_VIG;
428  VariableMapVector oldVars = pano.getVariables();
429  optimizePhotometric(pano,
430  createOptVars(pano, vars, opts.colorReferenceImage),
431  correspondences, imageStepSize, progress, error);
432  // check if vignetting is plausible
433  if(IsHighVignetting(pano.getImage(0).getRadialVigCorrCoeff()))
434  {
435  vars &= ~OPT_VIG;
436  pano.updateVariables(oldVars);
437  };
438  };
439 
440  // now take response curve into account
441  vars |= OPT_RESP;
442  // also WB if desired
443  if (mode == OPT_PHOTOMETRIC_LDR_WB || mode == OPT_PHOTOMETRIC_HDR_WB)
444  {
445  vars |= OPT_WB;
446  };
447  VariableMapVector oldVars = pano.getVariables();
448  optimizePhotometric(pano,
449  createOptVars(pano, vars, opts.colorReferenceImage),
450  correspondences, imageStepSize, progress, error);
451  // now check the results
452  const bool hasHighVignetting = IsHighVignetting(pano.getImage(0).getRadialVigCorrCoeff());
453  // @TODO check also response curve, what parameters are considered invalid?
454  const bool hasHighWB = CheckStrangeWB(pano);
455  if(hasHighVignetting)
456  {
457  vars &= ~OPT_VIG;
458  };
459  if(hasHighWB)
460  {
461  vars &= ~OPT_WB;
462  };
463  if(hasHighVignetting || hasHighWB)
464  {
465  // we got strange results, optimize again with less parameters
466  pano.updateVariables(oldVars);
467  if(vars>0)
468  {
469  optimizePhotometric(pano,
470  createOptVars(pano, vars, opts.colorReferenceImage),
471  correspondences, imageStepSize, progress, error);
472  };
473  };
474 }
475 
476 
478 {
482 
483  // optimizePhotometric does not tell us if it's cancelled
485  {
486  cancelAlgorithm();
487  }
488 
489  return wasCancelled(); // let's hope so.
490 }
491 
493 {
494  smartOptimizePhotometric(o_panorama,
495  o_optMode,
498 
499  // smartOptimizePhotometric does not tell us if it's cancelled
501  {
502  cancelAlgorithm();
503  };
504 
505  return !wasCancelled(); // let's hope so.
506 }
507 
508 } //namespace
static int photometricVis(double *p, double *x, int m, int n, int iter, double sqerror, void *data)
std::vector< UIntSet > getHDRStacks(const PanoramaData &pano, UIntSet allImgs, PanoramaOptions opts)
returns vector of set of output stacks
Definition: LayerStacks.cpp:35
declaration of functions to handle stacks and layers
static void photometricError(double *p, double *x, int m, int n, void *data)
void FromX(double *x)
copy new values from x to into this-&gt;m_imgs
virtual void printPanoramaScript(std::ostream &o, const OptimizeVector &optvars, const PanoramaOptions &options, const UIntSet &imgs, bool forPTOptimizer, const std::string &stripPrefix="") const =0
create an optimizer script
virtual void setSrcImage(unsigned int nr, const SrcPanoImage &img)=0
set input image parameters TODO: Propagate changes to linked images.
std::vector< float > EMoRParams
virtual void cancelAlgorithm()
Call this when the algorithm is cancelled.
bool CheckStrangeWB(PanoramaData &pano)
radiometric transformation, includes exposure, vignetting and white balance
hugin_utils::FDiff2D RadialVigCorrCenterShift
radiometric transformation, includes exposure, vignetting and white balance.
std::vector< double > RadialVigCorrCoeff
bool IsHighVignetting(std::vector< double > vigCorr)
bool set_contains(const _Container &c, const typename _Container::key_type &key)
Definition: stl_utils.h:74
double WhiteBalanceBlue
PhotometricOptimizeMode
local optimize definition.
bool updateDisplay()
updates the display, return true, if update was successful, false if cancel was pressed ...
vigra::pair< typename ROIImage< Image, Mask >::image_const_traverser, typename ROIImage< Image, Mask >::ImageConstAccessor > srcImage(const ROIImage< Image, Mask > &img)
Definition: ROIImage.h:300
double weightHuber(double x, double sigma)
expects the abs(error) values
#define CheckLinked(name)
void ToX(double *x)
copy optimisation variables into x
std::set< unsigned int > UIntSet
Definition: PanoramaData.h:51
virtual bool runAlgorithm()
implementation of the algorithm.
std::vector< VariableMap > VariableMapVector
static double sigma
empirical model of response
Definition: SrcPanoImage.h:100
virtual AppBase::ProgressDisplay * getProgressDisplay() const
virtual UIntSet getActiveImages() const =0
get active images
float pow(float a, double b)
Definition: utils.h:181
static void smartOptimizePhotometric(PanoramaData &pano, PhotometricOptimizeMode mode, const std::vector< vigra_ext::PointPairRGB > &correspondences, const float imageStepSize, AppBase::ProgressDisplay *progress, double &error)
use various heuristics to decide what to optimize.
virtual const PanoramaOptions & getOptions() const =0
returns the options for this panorama
double WhiteBalanceRed
Model for a panorama.
Definition: PanoramaData.h:81
virtual const SrcPanoImage & getImage(std::size_t nr) const =0
get a panorama image, counting starts with 0
static void optimizePhotometric(PanoramaData &pano, const OptimizeVector &vars, const PointPairs &correspondences, const float imageStepSize, AppBase::ProgressDisplay *progress, double &error)
double calcVigFactor(hugin_utils::FDiff2D d) const
vigra::RGBValue< T, RIDX, GIDX, BIDX > log(vigra::RGBValue< T, RIDX, GIDX, BIDX > const &v)
component-wise logarithm
Definition: utils.h:209
virtual bool runAlgorithm()
implementation of the algorithm.
options wxIntPtr wxIntPtr sortData std::vector< PanoInfo > * data
#define DEBUG_DEBUG(msg)
Definition: utils.h:68
virtual VariableMapVector getVariables() const =0
get variables of this panorama
void setSize(vigra::Size2D val)
Set the image size in pixels.
static void info(const char *fmt,...)
Definition: svm.cpp:95
std::vector< std::set< std::string > > OptimizeVector
void fill_set(_Container &c, typename _Container::key_type begin, typename _Container::key_type end)
Definition: stl_utils.h:81
std::vector< vigra_ext::PointPairRGB > m_data
virtual SrcPanoImage getSrcImage(unsigned imgNr) const =0
get a complete description of a source image
virtual std::size_t getNrOfImages() const =0
number of images.
All variables of a source image.
Definition: SrcPanoImage.h:194
Panorama image options.
double ExposureValue
virtual void updateVariables(const VariableMapVector &vars)=0
Set the variables.
OptimData(const PanoramaData &pano, const OptimizeVector &optvars, const std::vector< vigra_ext::PointPairRGB > &data, double mEstimatorSigma, bool symmetric, int maxIter, AppBase::ProgressDisplay *progress)