Computing a 4D Light Field from a 2D Image

% 
% Copyright 2010 Mitsubishi Electric Research Laboratories All Rights Reserved.
% 
% Permission to use, copy and modify this software and its documentation without fee for educational, research and non-profit purposes, is hereby granted, provided that the above copyright notice and the following three paragraphs appear in all copies.
% 
% To request permission to incorporate this software into commercial products contact: Vice President of Marketing and Business Development; Mitsubishi Electric Research Laboratories (MERL), 201 Broadway, Cambridge, MA 02139 or .
% 
% IN NO EVENT SHALL MERL BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF MERL HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
% 
% MERL SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND MERL HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS OR MODIFICATIONS.
% 



%==================================================================
% Matlab Code for SIGGRAPH 2007 paper
% "Dappled Photography: Mask Enhanced Cameras for Heterodyned Light Fields and Coded Aperture Refocusing"

% Author: Amit Agrawal, Ashok Veeraraghvan
% Contact: agrawal at merl dot com, vashok at umiacs dot umd dot edu

% The following code
%   -reads 2D input image captured from our Heterodyne Light Field Camera
%   -Computes the 4D light field from 2D image
%   -Computes refocused images by taking a slice of 4D light field


clear all;close all;clc;

% Set gray scale to 0 for color images. Will need more memory
GRAY_SCALE = 1


% The physical mask placed close to the sensor has 4 harmonics, therefore
% we will have 9 angular samples in the light field
nAngles = 9;
cAngles = (nAngles+1)/2;


% The fundamental frequency of the cosine in the mask in pixels
F1Y = 238;  F1X = 191;  %Cosine Frequency in Pixels from Calibration Image
F12X = floor(F1X/2);
F12Y = floor(F1Y/2);

%PhaseShift due to Mask In-Plane Translation wrt Sensor
phi1 = 300;  phi2 = 150;


%read 2D image
disp('Reading Input Image...');
I = double(imread('InputCones.png'));

if(GRAY_SCALE)
    %take green channel only
    I = I(:,:,2);
end


%make image odd size
I = I(1:end,1:end-1,:);

%find size of image
[m,n,CH] = size(I);


%Compute Spectral Tile Centers, Peak Strengths and Phase
for i = 1:nAngles
    for j = 1:nAngles
        CentY(i,j) = (m+1)/2 + (i-cAngles)*F1Y;
        CentX(i,j) = (n+1)/2 + (j-cAngles)*F1X;
        %Mat(i,j) = exp(-sqrt(-1)*((phi1*pi/180)*(i-cAngles) + (phi2*pi/180)*(j-cAngles)));
    end
end

Mat = ones(nAngles,nAngles);
% 20 is because we cannot have negative values in the mask. So strenght of
% DC component is 20 times that of harmonics
Mat(cAngles,cAngles) = Mat(cAngles,cAngles) * 20;


% Beginning of 4D light field computation
% do for all color channel

for ch = 1:CH

    disp('=================================');
    disp(sprintf('Processing channel %d',ch));

    % Find FFT of image
    disp('Computing FFT of 2D image');
    f = fftshift(fft2(I(:,:,ch)));


    %If you want to visaulize the FFT of input 2D image (Figure 8 of
    %paper), uncomment the next 2 lines
    %     figure;imshow(log10(abs(f)),[]);colormap gray;
    %     title('2D FFT of captured image (Figure 8 of paper). Note the spectral replicas');


    %Rearrange Tiles of 2D FFT into 4D Planes to obtain FFT of 4D Light-Field
    disp('Rearranging 2D FFT into 4D');
    for i = 1: nAngles
        for j = 1: nAngles
            FFT_LF(:,:,i,j) =  f( CentY(i,j)-F12Y:CentY(i,j)+F12Y, CentX(i,j)-F12X:CentX(i,j)+F12X)/Mat(i,j);
        end
    end
    clear f

    k = sqrt(-1);
    for i = 1:nAngles
        for j = 1:nAngles
            shift = (phi1*pi/180)*(i-cAngles) + (phi2*pi/180)*(j-cAngles);
            FFT_LF(:,:,i,j) = FFT_LF(:,:,i,j)*exp(k*shift);
        end
    end



    disp('Computing inverse 4D FFT');
    LF     =    ifftn(ifftshift(FFT_LF)); %Compute Light-Field by 4D Inverse FFT
    clear FFT_LF

    if(ch==1)
        LF_R = LF;
    elseif(ch==2)
        LF_G = LF;
    elseif(ch==3)
        LF_B = LF;
    end
    clear LF

end
clear I

%Now we have 4D light fiel
disp('Light Field Computed. Done...');
disp('==========================================');




% Digital Refocusing Code
% Take a 2D slice of 4D light field
% For refocusing, we only need the FFT of light field, not the light field

disp('Synthesizing Refocused Images by taking 2D slice of 4D Light Field');

if(GRAY_SCALE)
    FFT_LF_R = fftshift(fftn(LF_R));
    clear LF_R
else
    FFT_LF_R = fftshift(fftn(LF_R));
    clear LF_R

    FFT_LF_G = fftshift(fftn(LF_G));
    clear LF_G

    FFT_LF_B = fftshift(fftn(LF_B));
    clear LF_B
end


% height and width of refocused image
H = size(FFT_LF_R,1);
W = size(FFT_LF_R,2);


count = 0;
for theta = -14:14

    count = count + 1;

    disp('===============================================');
    disp(sprintf('Calculating New ReFocused Image: theta = %d',theta));

    if(GRAY_SCALE)
        RefocusedImage = Refocus2D(FFT_LF_R,[theta,theta]);
    else
        RefocusedImage = zeros(H,W,3);
        RefocusedImage(:,:,1) = Refocus2D(FFT_LF_R,[theta,theta]);
        RefocusedImage(:,:,2) = Refocus2D(FFT_LF_G,[theta,theta]);
        RefocusedImage(:,:,3) = Refocus2D(FFT_LF_B,[theta,theta]);
    end

    str = sprintf('RefocusedImage%03d.png',count);

    %Scale RefocusedImage in [0,255]
    RefocusedImage = RefocusedImage - min(RefocusedImage(:));
    RefocusedImage = 255*RefocusedImage/max(RefocusedImage(:));

    %write as png image
    clear tt
    for ii = 1:CH
        tt(:,:,ii) = fliplr(RefocusedImage(:,:,ii)');
    end
    imwrite(uint8(tt),str);

    disp(sprintf('Refocused image written as %s',str));

end

• Compute the 2D Fourier transform of the captured image.
• Rearrange the 81 spectral tiles into four dimensions.
• Compute the inverse 4D Fourier transform.